From 04883a6da43f460417b3e1add3a5807032557828 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 16 Jun 2020 19:32:56 +0300 Subject: BLD: check if std=c99 is really required --- setup.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 8178a6c51..2679015ad 100755 --- a/setup.py +++ b/setup.py @@ -189,7 +189,6 @@ def check_submodules(): for line in status.splitlines(): if line.startswith('-') or line.startswith('+'): raise ValueError('Submodule not clean: {}'.format(line)) - class concat_license_files(): @@ -235,19 +234,32 @@ def get_build_overrides(): from numpy.distutils.command.build_clib import build_clib from numpy.distutils.command.build_ext import build_ext - def _is_using_gcc(obj): - is_gcc = False + def _is_using_old_gcc(obj): + is_old_gcc = False if obj.compiler.compiler_type == 'unix': cc = sysconfig.get_config_var("CC") if not cc: cc = "" compiler_name = os.path.basename(cc) - is_gcc = "gcc" in compiler_name - return is_gcc + if "gcc" in compiler_name: + out = subprocess.run(['gcc', '-dM', '-E', '-'], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + for line in out.stdout.split(b'\n'): + if b'__STDC_VERSION__' in line: + # line is something like + # #define __STDC_VERSION__ 201710L + val = line.split(b' ')[-1] + if val < b'199901L': + # before c99, so we must add it ourselves + is_old_gcc = True + return is_old_gcc class new_build_clib(build_clib): def build_a_library(self, build_info, lib_name, libraries): - if _is_using_gcc(self): + if _is_using_old_gcc(self): args = build_info.get('extra_compiler_args') or [] args.append('-std=c99') build_info['extra_compiler_args'] = args @@ -255,7 +267,7 @@ def get_build_overrides(): class new_build_ext(build_ext): def build_extension(self, ext): - if _is_using_gcc(self): + if _is_using_old_gcc(self): if '-std=c99' not in ext.extra_compile_args: ext.extra_compile_args.append('-std=c99') build_ext.build_extension(self, ext) -- cgit v1.2.1 From eab6af0ae7773bf4dc1ee2653a59da2c274b8ff3 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 16 Jun 2020 22:58:18 +0300 Subject: BLD: use '-dumpversion' to get gcc version --- setup.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index 2679015ad..6219254f0 100755 --- a/setup.py +++ b/setup.py @@ -237,24 +237,16 @@ def get_build_overrides(): def _is_using_old_gcc(obj): is_old_gcc = False if obj.compiler.compiler_type == 'unix': - cc = sysconfig.get_config_var("CC") - if not cc: - cc = "" - compiler_name = os.path.basename(cc) - if "gcc" in compiler_name: - out = subprocess.run(['gcc', '-dM', '-E', '-'], - stdin=subprocess.DEVNULL, + cc = obj.compiler.compiler[0] + if cc == "gcc": + out = subprocess.run([cc, '-dumpversion'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - for line in out.stdout.split(b'\n'): - if b'__STDC_VERSION__' in line: - # line is something like - # #define __STDC_VERSION__ 201710L - val = line.split(b' ')[-1] - if val < b'199901L': - # before c99, so we must add it ourselves - is_old_gcc = True + ver = float(out.stdout) + if ver < 6: + # perhaps 5 is OK? + is_old_gcc = True return is_old_gcc class new_build_clib(build_clib): -- cgit v1.2.1 From 4142eba57b0bde071cb6d0718c88c6ddf7a6e906 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 17 Jun 2020 15:37:02 +0300 Subject: BUG: fixes failures (from review) --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6219254f0..aa2ad3913 100755 --- a/setup.py +++ b/setup.py @@ -238,13 +238,14 @@ def get_build_overrides(): is_old_gcc = False if obj.compiler.compiler_type == 'unix': cc = obj.compiler.compiler[0] - if cc == "gcc": + if "gcc" in cc: out = subprocess.run([cc, '-dumpversion'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - ver = float(out.stdout) - if ver < 6: + # will print something like b'4.2.1\n' + ver_parts = out.stdout.split(b'.') + if int(ver_parts[0]) < 6: # perhaps 5 is OK? is_old_gcc = True return is_old_gcc -- cgit v1.2.1 From 57fb47cc267eef54005f97d8e55b31df24590093 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 28 Jul 2020 06:39:16 +0300 Subject: MAINT: fixes from review --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index aa2ad3913..5dedf7a7b 100755 --- a/setup.py +++ b/setup.py @@ -233,6 +233,7 @@ def get_build_overrides(): """ from numpy.distutils.command.build_clib import build_clib from numpy.distutils.command.build_ext import build_ext + from distutils.version import LooseVersion def _is_using_old_gcc(obj): is_old_gcc = False @@ -242,11 +243,10 @@ def get_build_overrides(): out = subprocess.run([cc, '-dumpversion'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, ) - # will print something like b'4.2.1\n' - ver_parts = out.stdout.split(b'.') - if int(ver_parts[0]) < 6: - # perhaps 5 is OK? + # will print something like '4.2.1\n' + if LooseVersion(out.stdout) < LooseVersion('5.0'): is_old_gcc = True return is_old_gcc -- cgit v1.2.1 From 10dcfb0da453b103192a3adeca3197ac161f29a0 Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 28 Jul 2020 11:56:35 +0300 Subject: MAINT: refactoring from review --- setup.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 5dedf7a7b..a9296f1b9 100755 --- a/setup.py +++ b/setup.py @@ -235,24 +235,25 @@ def get_build_overrides(): from numpy.distutils.command.build_ext import build_ext from distutils.version import LooseVersion - def _is_using_old_gcc(obj): - is_old_gcc = False - if obj.compiler.compiler_type == 'unix': - cc = obj.compiler.compiler[0] - if "gcc" in cc: - out = subprocess.run([cc, '-dumpversion'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - # will print something like '4.2.1\n' - if LooseVersion(out.stdout) < LooseVersion('5.0'): - is_old_gcc = True - return is_old_gcc + def _needs_gcc_c99_flag(obj): + if obj.compiler.compiler_type != 'unix': + return False + + cc = obj.compiler.compiler[0] + if "gcc" not in cc: + return False + + # will print something like '4.2.1\n' + out = subprocess.run([cc, '-dumpversion'], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + # -std=c99 is default from this version on + if LooseVersion(out.stdout) >= LooseVersion('5.0'): + return False + return True class new_build_clib(build_clib): def build_a_library(self, build_info, lib_name, libraries): - if _is_using_old_gcc(self): + if _needs_gcc_c99_flag(self): args = build_info.get('extra_compiler_args') or [] args.append('-std=c99') build_info['extra_compiler_args'] = args @@ -260,7 +261,7 @@ def get_build_overrides(): class new_build_ext(build_ext): def build_extension(self, ext): - if _is_using_old_gcc(self): + if _needs_gcc_c99_flag(self): if '-std=c99' not in ext.extra_compile_args: ext.extra_compile_args.append('-std=c99') build_ext.build_extension(self, ext) -- cgit v1.2.1 From 6cb4d9b106c867b4bfb86b4abfd3dfc9e2709182 Mon Sep 17 00:00:00 2001 From: marload Date: Mon, 13 Jul 2020 18:14:13 +0900 Subject: BUG: linspace should round towards -infinity --- numpy/core/function_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index f57e95742..34818e334 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -160,6 +160,9 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, if axis != 0: y = _nx.moveaxis(y, 0, axis) + + if _nx.issubdtype(dtype, np.integer): + y[y<0] = _nx.floor(y[y<0]) if retstep: return y.astype(dtype, copy=False), step -- cgit v1.2.1 From 96a91d37b3b0f1452d0bdcf0071dd2a5a4079d56 Mon Sep 17 00:00:00 2001 From: Wansoo Kim Date: Mon, 13 Jul 2020 18:47:45 +0900 Subject: Update numpy/core/function_base.py Co-authored-by: Eric Wieser --- numpy/core/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 34818e334..7a2b350a4 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -162,7 +162,7 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, y = _nx.moveaxis(y, 0, axis) if _nx.issubdtype(dtype, np.integer): - y[y<0] = _nx.floor(y[y<0]) + _nx.floor(y, out=y) if retstep: return y.astype(dtype, copy=False), step -- cgit v1.2.1 From 020122fa08a82223324f1ac4f74c30b969da6e5b Mon Sep 17 00:00:00 2001 From: Wansoo Kim Date: Mon, 13 Jul 2020 18:47:57 +0900 Subject: Update numpy/core/function_base.py Co-authored-by: Eric Wieser --- numpy/core/function_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 7a2b350a4..ed0ad31bd 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -161,7 +161,7 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, if axis != 0: y = _nx.moveaxis(y, 0, axis) - if _nx.issubdtype(dtype, np.integer): + if _nx.issubdtype(dtype, _nx.integer): _nx.floor(y, out=y) if retstep: -- cgit v1.2.1 From 3cb6820c7e0b3ebf0afe10456aaadc1ba2618dea Mon Sep 17 00:00:00 2001 From: marload Date: Tue, 14 Jul 2020 12:35:31 +0900 Subject: Fix Test --- doc/release/upcoming_changes/16841.changes.rst | 4 ++++ numpy/core/tests/test_function_base.py | 7 +++++++ 2 files changed, 11 insertions(+) create mode 100644 doc/release/upcoming_changes/16841.changes.rst diff --git a/doc/release/upcoming_changes/16841.changes.rst b/doc/release/upcoming_changes/16841.changes.rst new file mode 100644 index 000000000..3408c091f --- /dev/null +++ b/doc/release/upcoming_changes/16841.changes.rst @@ -0,0 +1,4 @@ +Output if ``dtype`` of ``linspace`` is int +---------------------------------------------------- +- If the ``dtype`` of the ``linspace`` is int, round down it and casted it. +- To use it, write ``np.linspace(..., dtype=np.int)`` \ No newline at end of file diff --git a/numpy/core/tests/test_function_base.py b/numpy/core/tests/test_function_base.py index 62a9772c8..bc51b8144 100644 --- a/numpy/core/tests/test_function_base.py +++ b/numpy/core/tests/test_function_base.py @@ -402,3 +402,10 @@ class TestLinspace: stop = array(2, dtype='O') y = linspace(start, stop, 3) assert_array_equal(y, array([1., 1.5, 2.])) + + def test_round_negative(self): + y = linspace(-1, 3, num=8, dtype=int) + t = array([-1, -1, 0, 0, 1, 1, 2, 3], dtype=int) + assert_array_equal(y, t) + + -- cgit v1.2.1 From 1345286b67bea1efa873e79654f8a8609f1ce8f7 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sun, 2 Aug 2020 16:23:33 +0300 Subject: Update 16841.changes.rst reword the release note --- doc/release/upcoming_changes/16841.changes.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/release/upcoming_changes/16841.changes.rst b/doc/release/upcoming_changes/16841.changes.rst index 3408c091f..26efd2528 100644 --- a/doc/release/upcoming_changes/16841.changes.rst +++ b/doc/release/upcoming_changes/16841.changes.rst @@ -1,4 +1,5 @@ -Output if ``dtype`` of ``linspace`` is int ----------------------------------------------------- -- If the ``dtype`` of the ``linspace`` is int, round down it and casted it. -- To use it, write ``np.linspace(..., dtype=np.int)`` \ No newline at end of file +`np.linspace` on integers now use floor +--------------------------------------- +When using a `int` dtype in `np.linspace`, previously float values would +be rounded towards zero. Now `np.floor` is used instead, which rounds toward +``-inf``. This changes the results for negative values. -- cgit v1.2.1 From c7bf0311839a56461b4c9de0a2feb145a03b3705 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 3 Aug 2020 15:04:20 -0700 Subject: Update changelog name. --- doc/release/upcoming_changes/16841.change.rst | 5 +++++ doc/release/upcoming_changes/16841.changes.rst | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 doc/release/upcoming_changes/16841.change.rst delete mode 100644 doc/release/upcoming_changes/16841.changes.rst diff --git a/doc/release/upcoming_changes/16841.change.rst b/doc/release/upcoming_changes/16841.change.rst new file mode 100644 index 000000000..26efd2528 --- /dev/null +++ b/doc/release/upcoming_changes/16841.change.rst @@ -0,0 +1,5 @@ +`np.linspace` on integers now use floor +--------------------------------------- +When using a `int` dtype in `np.linspace`, previously float values would +be rounded towards zero. Now `np.floor` is used instead, which rounds toward +``-inf``. This changes the results for negative values. diff --git a/doc/release/upcoming_changes/16841.changes.rst b/doc/release/upcoming_changes/16841.changes.rst deleted file mode 100644 index 26efd2528..000000000 --- a/doc/release/upcoming_changes/16841.changes.rst +++ /dev/null @@ -1,5 +0,0 @@ -`np.linspace` on integers now use floor ---------------------------------------- -When using a `int` dtype in `np.linspace`, previously float values would -be rounded towards zero. Now `np.floor` is used instead, which rounds toward -``-inf``. This changes the results for negative values. -- cgit v1.2.1 From 8e914cf7f1c6433c8f7f76eccdb67375fece4126 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 3 Aug 2020 15:04:30 -0700 Subject: Update rst links in changelog --- doc/release/upcoming_changes/16841.change.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release/upcoming_changes/16841.change.rst b/doc/release/upcoming_changes/16841.change.rst index 26efd2528..9f112fd52 100644 --- a/doc/release/upcoming_changes/16841.change.rst +++ b/doc/release/upcoming_changes/16841.change.rst @@ -1,5 +1,5 @@ `np.linspace` on integers now use floor --------------------------------------- -When using a `int` dtype in `np.linspace`, previously float values would -be rounded towards zero. Now `np.floor` is used instead, which rounds toward +When using a `int` dtype in `numpy.linspace`, previously float values would +be rounded towards zero. Now `numpy.floor` is used instead, which rounds toward ``-inf``. This changes the results for negative values. -- cgit v1.2.1 From 1f2aef37cbaf83749699af6e80a33bd02b74b729 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 3 Aug 2020 16:08:14 -0700 Subject: DOC: Add example to release note --- doc/release/upcoming_changes/16841.change.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/release/upcoming_changes/16841.change.rst b/doc/release/upcoming_changes/16841.change.rst index 9f112fd52..d9499b6f4 100644 --- a/doc/release/upcoming_changes/16841.change.rst +++ b/doc/release/upcoming_changes/16841.change.rst @@ -2,4 +2,18 @@ --------------------------------------- When using a `int` dtype in `numpy.linspace`, previously float values would be rounded towards zero. Now `numpy.floor` is used instead, which rounds toward -``-inf``. This changes the results for negative values. +``-inf``. This changes the results for negative values. For example, the +following would previously give:: + + >>> np.linspace(-3, 1, 8, dtype=int) + array([-3, -2, -1, -1, 0, 0, 0, 1]) + +and now results in:: + + >>> np.linspace(-3, 1, 8, dtype=int) + array([-3, -3, -2, -2, -1, -1, 0, 1]) + +The former result can still be obtained with:: + + >>> np.linspace(-3, 1, 8).astype(int) + array([-3, -2, -1, -1, 0, 0, 0, 1]) -- cgit v1.2.1 From cd699831aa344fc1af8d8c91fb6e27eb41e35c00 Mon Sep 17 00:00:00 2001 From: Ross Barnowski Date: Mon, 3 Aug 2020 16:18:16 -0700 Subject: DOC: Added versionchanged to linspace docstring. --- numpy/core/function_base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index ed0ad31bd..46ef6d6ec 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -34,6 +34,10 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, .. versionchanged:: 1.16.0 Non-scalar `start` and `stop` are now supported. + .. versionchanged:: 1.20.0 + Values are rounded towards ``-inf`` instead of ``0`` when an + integer ``dtype`` is specified. + Parameters ---------- start : array_like -- cgit v1.2.1 From 962525096dce0717377141c423a080a3cc636f9f Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 4 Aug 2020 11:49:39 +0200 Subject: DOC: Update docs of clip() to allow for scalar arguments too --- numpy/core/fromnumeric.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index f8c11c015..2e8f29fc6 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -2057,7 +2057,7 @@ def clip(a, a_min, a_max, out=None, **kwargs): Parameters ---------- - a : array_like + a : scalar or array_like Array containing elements to clip. a_min : scalar or array_like or None Minimum value. If None, clipping is not performed on lower -- cgit v1.2.1 From 85c04b023e0320eda5a1e25fcbf8e5c45f9d05d7 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 4 Aug 2020 21:27:56 +0200 Subject: DOC: improve documentation of numpy.clip() after discussion --- numpy/core/fromnumeric.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 2e8f29fc6..924690e4b 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -2057,13 +2057,13 @@ def clip(a, a_min, a_max, out=None, **kwargs): Parameters ---------- - a : scalar or array_like + a : array_like Array containing elements to clip. - a_min : scalar or array_like or None + a_min : array_like or None Minimum value. If None, clipping is not performed on lower interval edge. Not more than one of `a_min` and `a_max` may be None. - a_max : scalar or array_like or None + a_max : array_like or None Maximum value. If None, clipping is not performed on upper interval edge. Not more than one of `a_min` and `a_max` may be None. If `a_min` or `a_max` are array_like, then the three -- cgit v1.2.1 From 6d37ebee366adb85767db7e924905c5f11f69e6d Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 4 Aug 2020 17:40:38 -0500 Subject: MAINT: Simplify scalar power This consolidates the multiple copies of almost identical scalar power code. It fixes the insignificant fact, that integers were doing some floating point flag handling, even though our scalar power does not use that. The previous code special cased powers of 0, but the C-function seems to handle it just fine. --- numpy/core/src/umath/scalarmath.c.src | 162 +++------------------------------- 1 file changed, 10 insertions(+), 152 deletions(-) diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index 90cc7a513..de00fc269 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -932,96 +932,14 @@ static PyObject * * Double, LongDouble, * CFloat, CDouble, CLongDouble# * - * #isint = (1,0)*5,0*7# + * #isint = 1*10,0*7# + * #isuint = (0,1)*5,0*7# * #cmplx = 0*14,1*3# * #iszero = _IS_ZERO*10, npy_half_iszero, _IS_ZERO*6# * #zero = 0*10, NPY_HALF_ZERO, 0*6# * #one = 1*10, NPY_HALF_ONE, 1*6# */ -#if @cmplx@ -static PyObject * -@name@_power(PyObject *a, PyObject *b, PyObject *modulo) -{ - PyObject *ret; - @type@ arg1, arg2; - int retstatus; - int first; - @type@ out = {@zero@, @zero@}; - - BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); - - switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { - case 0: - break; - case -1: - /* can't cast both safely mixed-types? */ - return PyArray_Type.tp_as_number->nb_power(a,b,modulo); - case -2: - /* use default handling */ - if (PyErr_Occurred()) { - return NULL; - } - return PyGenericArrType_Type.tp_as_number->nb_power(a,b,modulo); - case -3: - default: - /* - * special case for longdouble and clongdouble - * because they have a recursive getitem in their dtype - */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - if (modulo != Py_None) { - /* modular exponentiation is not implemented (gh-8804) */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - npy_clear_floatstatus_barrier((char*)&out); - - /* - * here we do the actual calculation with arg1 and arg2 - * as a function call. - */ - if (@iszero@(arg2.real) && @iszero@(arg2.imag)) { - out.real = @one@; - out.imag = @zero@; - } - else { - @name@_ctype_power(arg1, arg2, &out); - } - - /* Check status flag. If it is set, then look up what to do */ - retstatus = npy_get_floatstatus_barrier((char*)&out); - if (retstatus) { - int bufsize, errmask; - PyObject *errobj; - - if (PyUFunc_GetPyValues("@name@_scalars", &bufsize, &errmask, - &errobj) < 0) { - return NULL; - } - first = 1; - if (PyUFunc_handlefperr(errmask, errobj, retstatus, &first)) { - Py_XDECREF(errobj); - return NULL; - } - Py_XDECREF(errobj); - } - - ret = PyArrayScalar_New(@Name@); - if (ret == NULL) { - return NULL; - } - PyArrayScalar_ASSIGN(ret, @Name@, out); - - return ret; -} - -#elif @isint@ - static PyObject * @name@_power(PyObject *a, PyObject *b, PyObject *modulo) { @@ -1058,85 +976,25 @@ static PyObject * return Py_NotImplemented; } +#if !@isint@ npy_clear_floatstatus_barrier((char*)&out); - +#endif /* * here we do the actual calculation with arg1 and arg2 * as a function call. */ +#if @isint@ && !@isuint@ if (arg2 < 0) { PyErr_SetString(PyExc_ValueError, "Integers to negative integer powers are not allowed."); return NULL; } +#endif @name@_ctype_power(arg1, arg2, &out); - ret = PyArrayScalar_New(@Name@); - if (ret == NULL) { - return NULL; - } - PyArrayScalar_ASSIGN(ret, @Name@, out); - - return ret; -} - -#else - -static PyObject * -@name@_power(PyObject *a, PyObject *b, PyObject *modulo) -{ - PyObject *ret; - @type@ arg1, arg2; - int retstatus; - int first; - - @type@ out = @zero@; - - BINOP_GIVE_UP_IF_NEEDED(a, b, nb_power, @name@_power); - - switch(_@name@_convert2_to_ctypes(a, &arg1, b, &arg2)) { - case 0: - break; - case -1: - /* can't cast both safely mixed-types? */ - return PyArray_Type.tp_as_number->nb_power(a,b,modulo); - case -2: - /* use default handling */ - if (PyErr_Occurred()) { - return NULL; - } - return PyGenericArrType_Type.tp_as_number->nb_power(a,b,modulo); - case -3: - default: - /* - * special case for longdouble and clongdouble - * because they have a recursive getitem in their dtype - */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - if (modulo != Py_None) { - /* modular exponentiation is not implemented (gh-8804) */ - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - - npy_clear_floatstatus_barrier((char*)&out); - - /* - * here we do the actual calculation with arg1 and arg2 - * as a function call. - */ - if (@iszero@(arg2)) { - out = @one@; - } - else { - @name@_ctype_power(arg1, arg2, &out); - } - +#if !@isint@ /* Check status flag. If it is set, then look up what to do */ - retstatus = npy_get_floatstatus_barrier((char*)&out); + int retstatus = npy_get_floatstatus_barrier((char*)&out); if (retstatus) { int bufsize, errmask; PyObject *errobj; @@ -1145,13 +1003,14 @@ static PyObject * &errobj) < 0) { return NULL; } - first = 1; + int first = 1; if (PyUFunc_handlefperr(errmask, errobj, retstatus, &first)) { Py_XDECREF(errobj); return NULL; } Py_XDECREF(errobj); } +#endif ret = PyArrayScalar_New(@Name@); if (ret == NULL) { @@ -1162,7 +1021,6 @@ static PyObject * return ret; } -#endif /**end repeat**/ #undef _IS_ZERO -- cgit v1.2.1 From 5d934a3b826e79abba84868f3fbcf7f7b6fea943 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 5 Aug 2020 09:54:28 -0500 Subject: MAINT: Remove volatile and comment from scalar operators The volatile statement was designed to prevent reordering of floating point error checks, however, this was more generally fixed in gh-11036, thus removing the need for the volatile declaration (and bringing the code in line with the rest of the file). --- numpy/core/src/umath/scalarmath.c.src | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index de00fc269..55bc958cb 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -794,15 +794,8 @@ static PyObject * { PyObject *ret; @type@ arg1, arg2; - /* - * NOTE: In gcc >= 4.1, the compiler will reorder floating point - * operations and floating point error state checks. In - * particular, the arithmetic operations were being reordered - * so that the errors weren't caught. Declaring this output - * variable volatile was the minimal fix for the issue. - * (Ticket #1671) - */ - volatile @otype@ out; + @otype@ out; + #if @twoout@ @otype@ out2; PyObject *obj; -- cgit v1.2.1 From e2c6fd02a2f0819e690918e3d4a0079151492828 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2020 09:00:53 +0000 Subject: MAINT: Bump hypothesis from 5.23.9 to 5.23.12 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.23.9 to 5.23.12. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.23.9...hypothesis-python-5.23.12) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 51551d01f..88ee0c64e 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.23.9 +hypothesis==5.23.12 pytest==6.0.1 pytz==2020.1 pytest-cov==2.10.0 -- cgit v1.2.1 From 49eaea7184cf524ff21ca9d88e806247577be543 Mon Sep 17 00:00:00 2001 From: danielhrisca Date: Tue, 11 Aug 2020 11:28:38 +0300 Subject: STY: core._internal * removed redundant parenthesis * use f-strings * use dict.items in _makenames_list --- numpy/core/_internal.py | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 85853622a..449926f58 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -17,26 +17,25 @@ except ImportError: IS_PYPY = platform.python_implementation() == 'PyPy' -if (sys.byteorder == 'little'): +if sys.byteorder == 'little': _nbo = '<' else: _nbo = '>' def _makenames_list(adict, align): allfields = [] - fnames = list(adict.keys()) - for fname in fnames: - obj = adict[fname] + + for fname, obj in adict.items(): n = len(obj) - if not isinstance(obj, tuple) or n not in [2, 3]: + if not isinstance(obj, tuple) or n not in (2, 3): raise ValueError("entry not a 2- or 3- tuple") - if (n > 2) and (obj[2] == fname): + if n > 2 and obj[2] == fname: continue num = int(obj[1]) - if (num < 0): + if num < 0: raise ValueError("invalid offset.") format = dtype(obj[0], align=align) - if (n > 2): + if n > 2: title = obj[2] else: title = None @@ -68,7 +67,7 @@ def _usefields(adict, align): res = adict[name] formats.append(res[0]) offsets.append(res[1]) - if (len(res) > 2): + if len(res) > 2: titles.append(res[2]) else: titles.append(None) @@ -108,7 +107,7 @@ def _array_descr(descriptor): for field in ordered_fields: if field[1] > offset: num = field[1] - offset - result.append(('', '|V%d' % num)) + result.append(('', f'|V{num}')) offset += num elif field[1] < offset: raise ValueError( @@ -128,7 +127,7 @@ def _array_descr(descriptor): if descriptor.itemsize > offset: num = descriptor.itemsize - offset - result.append(('', '|V%d' % num)) + result.append(('', f'|V{num}')) return result @@ -191,7 +190,7 @@ def _commastring(astr): (order1, order2)) order = order1 - if order in ['|', '=', _nbo]: + if order in ('|', '=', _nbo): order = '' dtype = order + dtype if (repeats == ''): @@ -223,7 +222,7 @@ def _getintp_ctype(): val = dummy_ctype(np.intp) else: char = dtype('p').char - if (char == 'i'): + if char == 'i': val = ctypes.c_int elif char == 'l': val = ctypes.c_long @@ -379,7 +378,7 @@ def _newnames(datatype, order): raise ValueError(f"unknown field name: {name}") from None seen.add(name) return tuple(list(order) + nameslist) - raise ValueError("unsupported order value: %s" % (order,)) + raise ValueError(f"unsupported order value: {order}") def _copy_fields(ary): """Return copy of structured array with padding between fields removed. @@ -680,8 +679,7 @@ def __dtype_from_pep3118(stream, is_subdtype): if not (is_padding and name is None): if name is not None and name in field_spec['names']: - raise RuntimeError("Duplicate field name '%s' in PEP3118 format" - % name) + raise RuntimeError(f"Duplicate field name '{name}' in PEP3118 format") field_spec['names'].append(name) field_spec['formats'].append(value) field_spec['offsets'].append(offset) @@ -717,7 +715,7 @@ def _fix_names(field_spec): j = 0 while True: - name = 'f{}'.format(j) + name = f'f{j}' if name not in names: break j = j + 1 @@ -790,7 +788,7 @@ def _ufunc_doc_signature_formatter(ufunc): if ufunc.nin == 1: in_args = 'x' else: - in_args = ', '.join('x{}'.format(i+1) for i in range(ufunc.nin)) + in_args = ', '.join(f'x{i+1}' for i in range(ufunc.nin)) # output arguments are both keyword or positional if ufunc.nout == 0: -- cgit v1.2.1 From 7127cdfb8553030ba317455c9f31fe4d7ed74e81 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 11 Aug 2020 15:53:55 +0200 Subject: ENH: Use elementwise comparisons with 0 rather than boolean casting --- doc/release/upcoming_changes/16911.deprecation.rst | 2 +- numpy/core/tests/test_deprecations.py | 12 ++++++++---- numpy/lib/function_base.py | 8 ++++++-- numpy/lib/tests/test_function_base.py | 3 +-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/doc/release/upcoming_changes/16911.deprecation.rst b/doc/release/upcoming_changes/16911.deprecation.rst index d4dcb629c..8f38ed989 100644 --- a/doc/release/upcoming_changes/16911.deprecation.rst +++ b/doc/release/upcoming_changes/16911.deprecation.rst @@ -4,4 +4,4 @@ The ``trim_zeros`` function will, in the future, require an array with the following two properties: * It must be 1D. -* It must be convertable into a boolean array. +* It must support elementwise comparisons with zero. diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index 9004bef30..f0eac24ee 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -710,15 +710,19 @@ class TestRaggedArray(_DeprecationTestCase): class TestTrimZeros(_DeprecationTestCase): # Numpy 1.20.0, 2020-07-31 - @pytest.mark.parametrize("arr", [np.random.rand(10, 10).tolist(), - np.random.rand(10).astype(str)]) - def test_deprecated(self, arr): + @pytest.mark.parametrize( + "arr,exc_type", + [(np.random.rand(10, 10).tolist(), ValueError), + (np.random.rand(10).astype(str), FutureWarning)] + ) + def test_deprecated(self, arr, exc_type): with warnings.catch_warnings(): warnings.simplefilter('error', DeprecationWarning) try: np.trim_zeros(arr) except DeprecationWarning as ex: - assert_(isinstance(ex.__cause__, ValueError)) + ex_cause = ex.__cause__ + assert_(isinstance(ex_cause, exc_type)) else: raise AssertionError("No error raised during function call") diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index cd8862c94..96e1c6de9 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1631,7 +1631,7 @@ def trim_zeros(filt, trim='fb'): # Numpy 1.20.0, 2020-07-31 warning = DeprecationWarning( "in the future trim_zeros will require a 1-D array as input " - "that is compatible with ndarray.astype(bool)" + "that supports elementwise comparisons with zero" ) warning.__cause__ = ex warnings.warn(warning, stacklevel=3) @@ -1643,7 +1643,11 @@ def trim_zeros(filt, trim='fb'): def _trim_zeros_new(filt, trim='fb'): """Newer optimized implementation of ``trim_zeros()``.""" - arr = np.asanyarray(filt).astype(bool, copy=False) + arr_any = np.asanyarray(filt) + with warnings.catch_warnings(): + # not all dtypes support elementwise comparisons with `0` (e.g. str) + warnings.simplefilter('error', FutureWarning) + arr = arr_any != 0 if arr_any.dtype != bool else arr_any if arr.ndim != 1: raise ValueError('trim_zeros requires an array of exactly one dimension') diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 89c1a2d9b..744034e01 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1169,10 +1169,9 @@ class TestTrimZeros: a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) b = a.astype(float) c = a.astype(complex) - d = np.array([None, [], 1, False, 'b', 3.0, range(4), b''], dtype=object) def values(self): - attr_names = ('a', 'b', 'c', 'd') + attr_names = ('a', 'b', 'c') return (getattr(self, name) for name in attr_names) def test_basic(self): -- cgit v1.2.1 From e62d5d15f3b8df247dcb133eaaceb71f5f60d8c0 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Tue, 11 Aug 2020 16:17:50 +0200 Subject: MAINT: Catching warnings is expensive; remove it --- numpy/lib/function_base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 96e1c6de9..e9fe936e9 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1644,12 +1644,13 @@ def trim_zeros(filt, trim='fb'): def _trim_zeros_new(filt, trim='fb'): """Newer optimized implementation of ``trim_zeros()``.""" arr_any = np.asanyarray(filt) - with warnings.catch_warnings(): - # not all dtypes support elementwise comparisons with `0` (e.g. str) - warnings.simplefilter('error', FutureWarning) - arr = arr_any != 0 if arr_any.dtype != bool else arr_any + arr = arr_any != 0 if arr_any.dtype != bool else arr_any - if arr.ndim != 1: + if arr is False: + # not all dtypes support elementwise comparisons with `0` (e.g. str); + # they will return `False` instead + raise TypeError('elementwise comparison failed; unsupported data type') + elif arr.ndim != 1: raise ValueError('trim_zeros requires an array of exactly one dimension') elif not len(arr): return filt -- cgit v1.2.1 From 4d1c5d83d078f4002c5a1a77d71181abf85b02da Mon Sep 17 00:00:00 2001 From: stphnlyd Date: Wed, 12 Aug 2020 00:03:24 +0800 Subject: BUG: fix pickling of arrays larger than 2GiB (gh-17059) Closes gh-17045 --- numpy/core/src/multiarray/methods.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index a2db8042f..e0b36e80f 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1616,7 +1616,7 @@ _getlist_pkl(PyArrayObject *self) } while (iter->index < iter->size) { theobject = getitem(iter->dataptr, self); - PyList_SET_ITEM(list, (int) iter->index, theobject); + PyList_SET_ITEM(list, iter->index, theobject); PyArray_ITER_NEXT(iter); } Py_DECREF(iter); @@ -1636,7 +1636,7 @@ _setlist_pkl(PyArrayObject *self, PyObject *list) return -1; } while(iter->index < iter->size) { - theobject = PyList_GET_ITEM(list, (int) iter->index); + theobject = PyList_GET_ITEM(list, iter->index); setitem(theobject, iter->dataptr, self); PyArray_ITER_NEXT(iter); } -- cgit v1.2.1 From 6aa1360ff7d305144a6b0ce8eb838897c0e1091a Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:33:44 +0200 Subject: MAINT: Delete obsolete conversion to list (gh-17052) see gh-17012 item 12 --- numpy/distutils/command/build_ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index b6557fcf6..c7502d3e6 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -559,7 +559,7 @@ class build_ext (old_build_ext): unlinkable_fobjects = list(unlinkable_fobjects) # Expand possible fake static libraries to objects - for lib in list(libraries): + for lib in libraries: for libdir in library_dirs: fake_lib = os.path.join(libdir, lib + '.fobjects') if os.path.isfile(fake_lib): -- cgit v1.2.1 From 25fa87a92d39c4a7de72f79263c9048d657b1226 Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:34:40 +0200 Subject: MAINT: change "for line in open()" to "with open() as f: ..." in numpy/distutils/misc_util.py (gh-#17051) see #17012 item 13 Co-authored-by: Jakob --- numpy/distutils/misc_util.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index d2a3149f7..aa649a23f 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -1900,15 +1900,16 @@ class Configuration: revision0 = f.read().strip() branch_map = {} - for line in open(branch_cache_fn, 'r'): - branch1, revision1 = line.split()[:2] - if revision1==revision0: - branch0 = branch1 - try: - revision1 = int(revision1) - except ValueError: - continue - branch_map[branch1] = revision1 + with open(branch_cache_fn, 'r') as f: + for line in f: + branch1, revision1 = line.split()[:2] + if revision1==revision0: + branch0 = branch1 + try: + revision1 = int(revision1) + except ValueError: + continue + branch_map[branch1] = revision1 return branch_map.get(branch0) -- cgit v1.2.1 From c70bff83e452c9b76b7f5de2e6811b800d4f72cf Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:37:50 +0200 Subject: MAINT: Replace lambda function by list comprehension (gh-17055) See #17012 item 15 Co-authored-by: Jakob --- numpy/distutils/system_info.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index df82683dc..760bb7d5c 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -715,8 +715,7 @@ class system_info: AliasedOptionError : in case more than one of the options are found """ - found = map(lambda opt: self.cp.has_option(self.section, opt), options) - found = list(found) + found = [self.cp.has_option(self.section, opt) for opt in options] if sum(found) == 1: return options[found.index(True)] elif sum(found) == 0: -- cgit v1.2.1 From 7ec2e1bac72afcdc68cf8256879afbc4cb14a907 Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Tue, 11 Aug 2020 20:45:04 +0200 Subject: MAINT: Remove obsolete conversion to set (#17063) compare already returns sets, so conversion is redundant --- tools/refguide_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/refguide_check.py b/tools/refguide_check.py index b3891c5cb..43051d7e1 100644 --- a/tools/refguide_check.py +++ b/tools/refguide_check.py @@ -388,8 +388,8 @@ def check_items(all_dict, names, deprecated, others, module_name, dots=True): output += "Objects in refguide: %i\n\n" % num_ref only_all, only_ref, missing = compare(all_dict, others, names, module_name) - dep_in_ref = set(only_ref).intersection(deprecated) - only_ref = set(only_ref).difference(deprecated) + dep_in_ref = only_ref.intersection(deprecated) + only_ref = only_ref.difference(deprecated) if len(dep_in_ref) > 0: output += "Deprecated objects in refguide::\n\n" -- cgit v1.2.1 From 37dc69cd01934b78bbf9bcccab5fbc4a7ff5eacb Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 7 Aug 2020 16:08:26 -0500 Subject: MAINT: Add error return to all casting functionality and NpyIter NpyIter cannot properly indicate errors right away, defering them in some case until the actual cleanup occurs. to make that slightly with easier, this adds a new API function with (non-standard for NpyIter) error return as -1. A second function is one which only checks whether an error occurred (i.e. `NpyIter_ErrOccurred()`) which is necessary in the (more rare) case the iterator is not deallocated promptly after the iteration is finished. --- numpy/core/src/common/lowlevel_strided_loops.h | 37 +-- numpy/core/src/multiarray/array_assign_array.c | 15 +- numpy/core/src/multiarray/array_assign_scalar.c | 15 +- numpy/core/src/multiarray/array_coercion.c | 6 +- numpy/core/src/multiarray/ctors.c | 23 +- numpy/core/src/multiarray/dtype_transfer.c | 347 +++++++++++++-------- .../src/multiarray/lowlevel_strided_loops.c.src | 173 ++++++---- numpy/core/src/multiarray/mapping.c | 43 +-- numpy/core/src/multiarray/nditer_api.c | 182 +++++++++-- numpy/core/src/multiarray/nditer_constr.c | 43 ++- numpy/core/src/multiarray/nditer_impl.h | 7 +- numpy/core/src/multiarray/nditer_pywrap.c | 8 + numpy/core/src/multiarray/nditer_templ.c.src | 20 +- numpy/core/src/umath/reduction.c | 7 +- numpy/core/src/umath/ufunc_object.c | 36 ++- 15 files changed, 629 insertions(+), 333 deletions(-) diff --git a/numpy/core/src/common/lowlevel_strided_loops.h b/numpy/core/src/common/lowlevel_strided_loops.h index f2f12a55b..12aa61822 100644 --- a/numpy/core/src/common/lowlevel_strided_loops.h +++ b/numpy/core/src/common/lowlevel_strided_loops.h @@ -30,10 +30,9 @@ * Use NPY_AUXDATA_CLONE and NPY_AUXDATA_FREE to deal with this data. * */ -typedef void (PyArray_StridedUnaryOp)(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *transferdata); +typedef int (PyArray_StridedUnaryOp)( + char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, + npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata); /* * This is for pointers to functions which behave exactly as @@ -43,31 +42,10 @@ typedef void (PyArray_StridedUnaryOp)(char *dst, npy_intp dst_stride, * In particular, the 'i'-th element is operated on if and only if * mask[i*mask_stride] is true. */ -typedef void (PyArray_MaskedStridedUnaryOp)(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_bool *mask, npy_intp mask_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *transferdata); - -/* - * This function pointer is for binary operations that input two - * arbitrarily strided one-dimensional array segments and output - * an arbitrarily strided array segment of the same size. - * It may be a fully general function, or a specialized function - * when the strides or item size have particular known values. - * - * Examples of binary operations are the basic arithmetic operations, - * logical operators AND, OR, and many others. - * - * The 'transferdata' parameter is slightly special, following a - * generic auxiliary data pattern defined in ndarraytypes.h - * Use NPY_AUXDATA_CLONE and NPY_AUXDATA_FREE to deal with this data. - * - */ -typedef void (PyArray_StridedBinaryOp)(char *dst, npy_intp dst_stride, - char *src0, npy_intp src0_stride, - char *src1, npy_intp src1_stride, - npy_intp N, NpyAuxData *transferdata); +typedef int (PyArray_MaskedStridedUnaryOp)( + char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, + npy_bool *mask, npy_intp mask_stride, + npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata); /* * Gives back a function pointer to a specialized function for copying @@ -271,6 +249,7 @@ PyArray_CastRawArrays(npy_intp count, * The return value is the number of elements it couldn't copy. A return value * of 0 means all elements were copied, a larger value means the end of * the n-dimensional array was reached before 'count' elements were copied. + * A negative return value indicates an error occurred. * * ndim: * The number of dimensions of the n-dimensional array. diff --git a/numpy/core/src/multiarray/array_assign_array.c b/numpy/core/src/multiarray/array_assign_array.c index b8dc7d516..361964a5c 100644 --- a/numpy/core/src/multiarray/array_assign_array.c +++ b/numpy/core/src/multiarray/array_assign_array.c @@ -132,17 +132,22 @@ raw_array_assign_array(int ndim, npy_intp const *shape, NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ - stransfer(dst_data, dst_strides_it[0], src_data, src_strides_it[0], - shape_it[0], src_itemsize, transferdata); + if (stransfer( + dst_data, dst_strides_it[0], src_data, src_strides_it[0], + shape_it[0], src_itemsize, transferdata) < 0) { + goto fail; + } } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it, src_data, src_strides_it); NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); - - return (needs_api && PyErr_Occurred()) ? -1 : 0; + return 0; +fail: + NPY_END_THREADS; + NPY_AUXDATA_FREE(transferdata); + return -1; } /* diff --git a/numpy/core/src/multiarray/array_assign_scalar.c b/numpy/core/src/multiarray/array_assign_scalar.c index 41eb75f1c..023772776 100644 --- a/numpy/core/src/multiarray/array_assign_scalar.c +++ b/numpy/core/src/multiarray/array_assign_scalar.c @@ -82,16 +82,21 @@ raw_array_assign_scalar(int ndim, npy_intp const *shape, NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ - stransfer(dst_data, dst_strides_it[0], src_data, 0, - shape_it[0], src_itemsize, transferdata); + if (stransfer( + dst_data, dst_strides_it[0], src_data, 0, + shape_it[0], src_itemsize, transferdata) < 0) { + goto fail; + } } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it); NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); - - return (needs_api && PyErr_Occurred()) ? -1 : 0; + return 0; +fail: + NPY_END_THREADS; + NPY_AUXDATA_FREE(transferdata); + return -1; } /* diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 382645ff5..ffb5bd632 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -495,12 +495,10 @@ PyArray_Pack(PyArray_Descr *descr, char *item, PyObject *value) res = -1; goto finish; } - stransfer(item, 0, data, 0, 1, tmp_descr->elsize, transferdata); - NPY_AUXDATA_FREE(transferdata); - - if (needs_api && PyErr_Occurred()) { + if (stransfer(item, 0, data, 0, 1, tmp_descr->elsize, transferdata) < 0) { res = -1; } + NPY_AUXDATA_FREE(transferdata); finish: if (PyDataType_REFCHK(tmp_descr)) { diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index dc451685a..15824e9e2 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -2391,16 +2391,21 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) src_count = *src_countptr; dst_data = dst_dataptr[0]; src_data = src_dataptr[0]; + int res = 0; for(;;) { /* Transfer the biggest amount that fits both */ count = (src_count < dst_count) ? src_count : dst_count; - stransfer(dst_data, dst_stride, - src_data, src_stride, - count, src_itemsize, transferdata); + if (stransfer( + dst_data, dst_stride, src_data, src_stride, + count, src_itemsize, transferdata) < 0) { + res = -1; + break; + } /* If we exhausted the dst block, refresh it */ if (dst_count == count) { - if (!dst_iternext(dst_iter)) { + res = dst_iternext(dst_iter); + if (!res) { break; } dst_count = *dst_countptr; @@ -2413,7 +2418,8 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) /* If we exhausted the src block, refresh it */ if (src_count == count) { - if (!src_iternext(src_iter)) { + res = src_iternext(src_iter); + if (!res) { break; } src_count = *src_countptr; @@ -2430,8 +2436,11 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) NPY_AUXDATA_FREE(transferdata); NpyIter_Deallocate(dst_iter); NpyIter_Deallocate(src_iter); - - return PyErr_Occurred() ? -1 : 0; + if (res > 0) { + /* The iteration stopped successfully, do not report an error */ + return 0; + } + return res; } /*NUMPY_API diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 3a58b5849..c9868a2c8 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -106,7 +106,7 @@ get_bool_setdstone_transfer_function(npy_intp dst_stride, /*************************** COPY REFERENCES *******************************/ /* Moves references from src to dst */ -static void +static int _strided_to_strided_move_references(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -131,10 +131,11 @@ _strided_to_strided_move_references(char *dst, npy_intp dst_stride, dst += dst_stride; --N; } + return 0; } /* Copies references from src to dst */ -static void +static int _strided_to_strided_copy_references(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -158,6 +159,7 @@ _strided_to_strided_copy_references(char *dst, npy_intp dst_stride, dst += dst_stride; --N; } + return 0; } @@ -188,7 +190,7 @@ static NpyAuxData *_strided_zero_pad_data_clone(NpyAuxData *data) * Does a strided to strided zero-padded copy for the case where * dst_itemsize > src_itemsize */ -static void +static int _strided_to_strided_zero_pad_copy(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -205,13 +207,14 @@ _strided_to_strided_zero_pad_copy(char *dst, npy_intp dst_stride, dst += dst_stride; --N; } + return 0; } /* * Does a strided to strided zero-padded copy for the case where * dst_itemsize < src_itemsize */ -static void +static int _strided_to_strided_truncate_copy(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -226,13 +229,14 @@ _strided_to_strided_truncate_copy(char *dst, npy_intp dst_stride, dst += dst_stride; --N; } + return 0; } /* * Does a strided to strided zero-padded or truncated copy for the case where * unicode swapping is needed. */ -static void +static int _strided_to_strided_unicode_copyswap(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -260,6 +264,7 @@ _strided_to_strided_unicode_copyswap(char *dst, npy_intp dst_stride, dst += dst_stride; --N; } + return 0; } @@ -379,7 +384,7 @@ static NpyAuxData *_align_wrap_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -395,47 +400,50 @@ _strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, *todata = d->todata, *fromdata = d->fromdata; char *bufferin = d->bufferin, *bufferout = d->bufferout; - npy_bool init_dest = d->init_dest, out_needs_api = d->out_needs_api; + npy_bool init_dest = d->init_dest; for(;;) { - /* - * The caller does not know if a previous call resulted in a Python - * exception. Much of the Python API is unsafe while an exception is in - * flight, so just skip all the work. Someone higher in the call stack - * will check for errors and propagate them. - */ - if (out_needs_api && PyErr_Occurred()) { - return; - } if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { - tobuffer(bufferin, inner_src_itemsize, src, src_stride, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - src_itemsize, todata); + if (tobuffer( + bufferin, inner_src_itemsize, src, src_stride, + NPY_LOWLEVEL_BUFFER_BLOCKSIZE, src_itemsize, todata) < 0) { + return -1; + } if (init_dest) { memset(bufferout, 0, - dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE); + dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE); + } + if (wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, + NPY_LOWLEVEL_BUFFER_BLOCKSIZE, + inner_src_itemsize, wrappeddata) < 0) { + return -1; + } + if (frombuffer(dst, dst_stride, bufferout, dst_itemsize, + NPY_LOWLEVEL_BUFFER_BLOCKSIZE, + dst_itemsize, fromdata) < 0) { + return -1; } - wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - inner_src_itemsize, wrappeddata); - frombuffer(dst, dst_stride, bufferout, dst_itemsize, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - dst_itemsize, fromdata); N -= NPY_LOWLEVEL_BUFFER_BLOCKSIZE; src += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_stride; dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride; } else { - tobuffer(bufferin, inner_src_itemsize, src, src_stride, N, - src_itemsize, todata); + if (tobuffer(bufferin, inner_src_itemsize, src, src_stride, + N, src_itemsize, todata) < 0) { + return -1; + } if (init_dest) { memset(bufferout, 0, dst_itemsize*N); } - wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, N, - inner_src_itemsize, wrappeddata); - frombuffer(dst, dst_stride, bufferout, dst_itemsize, N, - dst_itemsize, fromdata); - return; + if (wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, + N, inner_src_itemsize, wrappeddata) < 0) { + return -1; + } + if (frombuffer(dst, dst_stride, bufferout, dst_itemsize, + N, dst_itemsize, fromdata) < 0) { + return -1; + } + return 0; } } } @@ -538,7 +546,7 @@ static NpyAuxData *_wrap_copy_swap_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_wrap_copy_swap(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp NPY_UNUSED(src_itemsize), @@ -546,7 +554,9 @@ _strided_to_strided_wrap_copy_swap(char *dst, npy_intp dst_stride, { _wrap_copy_swap_data *d = (_wrap_copy_swap_data *)data; + /* We assume that d->copyswapn should not be able to error. */ d->copyswapn(dst, dst_stride, src, src_stride, N, d->swap, d->arr); + return 0; } /* This only gets used for custom data types and for Unicode when swapping */ @@ -603,6 +613,7 @@ typedef struct { NpyAuxData base; PyArray_VectorUnaryFunc *castfunc; PyArrayObject *aip, *aop; + npy_bool needs_api; } _strided_cast_data; /* strided cast data free function */ @@ -630,7 +641,7 @@ static NpyAuxData *_strided_cast_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -639,17 +650,29 @@ _aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride, _strided_cast_data *d = (_strided_cast_data *)data; PyArray_VectorUnaryFunc *castfunc = d->castfunc; PyArrayObject *aip = d->aip, *aop = d->aop; + npy_bool needs_api = d->needs_api; while (N > 0) { castfunc(src, dst, 1, aip, aop); + /* + * Since error handling in ufuncs is not ideal (at the time of + * writing this, an error could be in process before calling this + * function. For most of NumPy history these checks were completely + * missing, so this is hopefully OK for the time being (until ufuncs + * are fixed). + */ + if (needs_api && PyErr_Occurred()) { + return -1; + } dst += dst_stride; src += src_stride; --N; } + return 0; } /* This one requires src be of type NPY_OBJECT */ -static void +static int _aligned_strided_to_strided_cast_decref_src(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -658,31 +681,49 @@ _aligned_strided_to_strided_cast_decref_src(char *dst, npy_intp dst_stride, _strided_cast_data *d = (_strided_cast_data *)data; PyArray_VectorUnaryFunc *castfunc = d->castfunc; PyArrayObject *aip = d->aip, *aop = d->aop; + npy_bool needs_api = d->needs_api; PyObject *src_ref; while (N > 0) { castfunc(src, dst, 1, aip, aop); - - /* After casting, decrement the source ref */ + /* + * See comment in `_aligned_strided_to_strided_cast`, an error could + * in principle be set before `castfunc` is called. + */ + if (needs_api && PyErr_Occurred()) { + return -1; + } + /* After casting, decrement the source ref and set it to NULL */ NPY_COPY_PYOBJECT_PTR(&src_ref, src); - NPY_DT_DBG_REFTRACE("dec src ref (cast object -> not object)", src_ref); Py_XDECREF(src_ref); + memset(src, 0, sizeof(PyObject *)); + NPY_DT_DBG_REFTRACE("dec src ref (cast object -> not object)", src_ref); dst += dst_stride; src += src_stride; --N; } + return 0; } -static void +static int _aligned_contig_to_contig_cast(char *dst, npy_intp NPY_UNUSED(dst_stride), char *src, npy_intp NPY_UNUSED(src_stride), npy_intp N, npy_intp NPY_UNUSED(itemsize), NpyAuxData *data) { _strided_cast_data *d = (_strided_cast_data *)data; + npy_bool needs_api = d->needs_api; d->castfunc(src, dst, N, d->aip, d->aop); + /* + * See comment in `_aligned_strided_to_strided_cast`, an error could + * in principle be set before `castfunc` is called. + */ + if (needs_api && PyErr_Occurred()) { + return -1; + } + return 0; } static int @@ -777,7 +818,7 @@ static NpyAuxData *_strided_datetime_cast_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -792,12 +833,12 @@ _strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride, if (convert_datetime_to_datetimestruct(&d->src_meta, dt, &dts) < 0) { - dt = NPY_DATETIME_NAT; + return -1; } else { if (convert_datetimestruct_to_datetime(&d->dst_meta, &dts, &dt) < 0) { - dt = NPY_DATETIME_NAT; + return -1; } } @@ -807,9 +848,10 @@ _strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride, src += src_stride; --N; } + return 0; } -static void +static int _strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -838,9 +880,10 @@ _strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, src += src_stride; --N; } + return 0; } -static void +static int _aligned_strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, @@ -870,9 +913,10 @@ _aligned_strided_to_strided_datetime_cast(char *dst, src += src_stride; --N; } + return 0; } -static void +static int _strided_to_strided_datetime_to_string(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp NPY_UNUSED(src_itemsize), @@ -888,28 +932,26 @@ _strided_to_strided_datetime_to_string(char *dst, npy_intp dst_stride, if (convert_datetime_to_datetimestruct(&d->src_meta, dt, &dts) < 0) { - /* For an error, produce a 'NaT' string */ - dts.year = NPY_DATETIME_NAT; + return -1; } /* Initialize the destination to all zeros */ memset(dst, 0, dst_itemsize); - /* - * This may also raise an error, but the caller needs - * to use PyErr_Occurred(). - */ - make_iso_8601_datetime(&dts, dst, dst_itemsize, + if (make_iso_8601_datetime(&dts, dst, dst_itemsize, 0, 0, d->src_meta.base, -1, - NPY_UNSAFE_CASTING); + NPY_UNSAFE_CASTING) < 0) { + return -1; + } dst += dst_stride; src += src_stride; --N; } + return 0; } -static void +static int _strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -934,7 +976,7 @@ _strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, if (parse_iso_8601_datetime(tmp_buffer, src_itemsize, d->dst_meta.base, NPY_SAME_KIND_CASTING, &dts, NULL, NULL) < 0) { - dt = NPY_DATETIME_NAT; + return -1; } } /* Otherwise parse the data in place */ @@ -942,7 +984,7 @@ _strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, if (parse_iso_8601_datetime(src, tmp - src, d->dst_meta.base, NPY_SAME_KIND_CASTING, &dts, NULL, NULL) < 0) { - dt = NPY_DATETIME_NAT; + return -1; } } @@ -950,7 +992,7 @@ _strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, if (dt != NPY_DATETIME_NAT && convert_datetimestruct_to_datetime(&d->dst_meta, &dts, &dt) < 0) { - dt = NPY_DATETIME_NAT; + return -1; } memcpy(dst, &dt, sizeof(dt)); @@ -959,6 +1001,7 @@ _strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, src += src_stride; --N; } + return 0; } /* @@ -1422,6 +1465,7 @@ get_nbo_cast_transfer_function(int aligned, data->base.free = &_strided_cast_data_free; data->base.clone = &_strided_cast_data_clone; data->castfunc = castfunc; + data->needs_api = *out_needs_api; /* * TODO: This is a hack so the cast functions have an array. * The cast functions shouldn't need that. Also, since we @@ -1652,7 +1696,7 @@ static NpyAuxData *_one_to_n_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_one_to_n(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -1664,18 +1708,19 @@ _strided_to_strided_one_to_n(char *dst, npy_intp dst_stride, npy_intp subN = d->N, dst_itemsize = d->dst_itemsize; while (N > 0) { - subtransfer(dst, dst_itemsize, - src, 0, - subN, src_itemsize, - subdata); + if (subtransfer( + dst, dst_itemsize, src, 0, subN, src_itemsize, subdata) < 0) { + return -1; + } src += src_stride; dst += dst_stride; --N; } + return 0; } -static void +static int _strided_to_strided_one_to_n_with_finish(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -1688,21 +1733,21 @@ _strided_to_strided_one_to_n_with_finish(char *dst, npy_intp dst_stride, npy_intp subN = d->N, dst_itemsize = d->dst_itemsize; while (N > 0) { - subtransfer(dst, dst_itemsize, - src, 0, - subN, src_itemsize, - subdata); - + if (subtransfer( + dst, dst_itemsize, src, 0, subN, src_itemsize, subdata) < 0) { + return -1; + } - stransfer_finish_src(NULL, 0, - src, 0, - 1, src_itemsize, - data_finish_src); + if (stransfer_finish_src( + NULL, 0, src, 0, 1, src_itemsize, data_finish_src) < 0) { + return -1; + } src += src_stride; dst += dst_stride; --N; } + return 0; } /* @@ -1846,7 +1891,7 @@ static NpyAuxData *_n_to_n_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_n_to_n(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -1859,18 +1904,19 @@ _strided_to_strided_n_to_n(char *dst, npy_intp dst_stride, dst_subitemsize = d->dst_itemsize; while (N > 0) { - subtransfer(dst, dst_subitemsize, - src, src_subitemsize, - subN, src_subitemsize, - subdata); - + if (subtransfer( + dst, dst_subitemsize, src, src_subitemsize, + subN, src_subitemsize, subdata) < 0) { + return -1; + } src += src_stride; dst += dst_stride; --N; } + return 0; } -static void +static int _contig_to_contig_n_to_n(char *dst, npy_intp NPY_UNUSED(dst_stride), char *src, npy_intp NPY_UNUSED(src_stride), npy_intp N, npy_intp NPY_UNUSED(src_itemsize), @@ -1882,10 +1928,12 @@ _contig_to_contig_n_to_n(char *dst, npy_intp NPY_UNUSED(dst_stride), npy_intp subN = d->N, src_subitemsize = d->src_itemsize, dst_subitemsize = d->dst_itemsize; - subtransfer(dst, dst_subitemsize, - src, src_subitemsize, - subN*N, src_subitemsize, - subdata); + if (subtransfer( + dst, dst_subitemsize, src, src_subitemsize, + subN*N, src_subitemsize, subdata) < 0) { + return -1; + } + return 0; } /* @@ -2049,7 +2097,7 @@ static NpyAuxData *_subarray_broadcast_data_clone( NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_subarray_broadcast(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp NPY_UNUSED(src_itemsize), @@ -2072,10 +2120,11 @@ _strided_to_strided_subarray_broadcast(char *dst, npy_intp dst_stride, count = offsetruns[run].count; dst_ptr = dst + loop_index*dst_subitemsize; if (offset != -1) { - subtransfer(dst_ptr, dst_subitemsize, - src + offset, src_subitemsize, - count, src_subitemsize, - subdata); + if (subtransfer( + dst_ptr, dst_subitemsize, src + offset, src_subitemsize, + count, src_subitemsize, subdata) < 0) { + return -1; + } } else { memset(dst_ptr, 0, count*dst_subitemsize); @@ -2087,10 +2136,11 @@ _strided_to_strided_subarray_broadcast(char *dst, npy_intp dst_stride, dst += dst_stride; --N; } + return 0; } -static void +static int _strided_to_strided_subarray_broadcast_withrefs(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp NPY_UNUSED(src_itemsize), @@ -2118,16 +2168,19 @@ _strided_to_strided_subarray_broadcast_withrefs(char *dst, npy_intp dst_stride, count = offsetruns[run].count; dst_ptr = dst + loop_index*dst_subitemsize; if (offset != -1) { - subtransfer(dst_ptr, dst_subitemsize, - src + offset, src_subitemsize, - count, src_subitemsize, - subdata); + if (subtransfer( + dst_ptr, dst_subitemsize, src + offset, src_subitemsize, + count, src_subitemsize, subdata) < 0) { + return -1; + } } else { if (stransfer_decdstref != NULL) { - stransfer_decdstref(NULL, 0, dst_ptr, dst_subitemsize, - count, dst_subitemsize, - data_decdstref); + if (stransfer_decdstref( + NULL, 0, dst_ptr, dst_subitemsize, + count, dst_subitemsize, data_decdstref) < 0) { + return -1; + } } memset(dst_ptr, 0, count*dst_subitemsize); } @@ -2135,15 +2188,18 @@ _strided_to_strided_subarray_broadcast_withrefs(char *dst, npy_intp dst_stride, } if (stransfer_decsrcref != NULL) { - stransfer_decsrcref(NULL, 0, src, src_subitemsize, - src_subN, src_subitemsize, - data_decsrcref); + if (stransfer_decsrcref( + NULL, 0, src, src_subitemsize, + src_subN, src_subitemsize, data_decsrcref) < 0) { + return -1; + } } src += src_stride; dst += dst_stride; --N; } + return 0; } @@ -2500,7 +2556,7 @@ static NpyAuxData *_field_transfer_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _strided_to_strided_field_transfer(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp NPY_UNUSED(src_itemsize), @@ -2515,11 +2571,13 @@ _strided_to_strided_field_transfer(char *dst, npy_intp dst_stride, field = &d->fields; if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { for (i = 0; i < field_count; ++i, ++field) { - field->stransfer(dst + field->dst_offset, dst_stride, - src + field->src_offset, src_stride, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - field->src_itemsize, - field->data); + if (field->stransfer( + dst + field->dst_offset, dst_stride, + src + field->src_offset, src_stride, + NPY_LOWLEVEL_BUFFER_BLOCKSIZE, + field->src_itemsize, field->data) < 0) { + return -1; + } } N -= NPY_LOWLEVEL_BUFFER_BLOCKSIZE; src += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_stride; @@ -2527,13 +2585,15 @@ _strided_to_strided_field_transfer(char *dst, npy_intp dst_stride, } else { for (i = 0; i < field_count; ++i, ++field) { - field->stransfer(dst + field->dst_offset, dst_stride, - src + field->src_offset, src_stride, - N, - field->src_itemsize, - field->data); + if (field->stransfer( + dst + field->dst_offset, dst_stride, + src + field->src_offset, src_stride, + N, + field->src_itemsize, field->data) < 0) { + return -1; + } } - return; + return 0; } } } @@ -2947,7 +3007,8 @@ static NpyAuxData *_masked_wrapper_transfer_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void _strided_masked_wrapper_decsrcref_transfer_function( +static int +_strided_masked_wrapper_decsrcref_transfer_function( char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_bool *mask, npy_intp mask_stride, @@ -2969,8 +3030,11 @@ static void _strided_masked_wrapper_decsrcref_transfer_function( /* Skip masked values, still calling decsrcref for move_references */ mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N, &subloopsize, 1); - decsrcref_stransfer(NULL, 0, src, src_stride, - subloopsize, src_itemsize, decsrcref_transferdata); + if (decsrcref_stransfer( + NULL, 0, src, src_stride, + subloopsize, src_itemsize, decsrcref_transferdata) < 0) { + return -1; + } dst += subloopsize * dst_stride; src += subloopsize * src_stride; N -= subloopsize; @@ -2981,15 +3045,20 @@ static void _strided_masked_wrapper_decsrcref_transfer_function( /* Process unmasked values */ mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N, &subloopsize, 0); - unmasked_stransfer(dst, dst_stride, src, src_stride, - subloopsize, src_itemsize, unmasked_transferdata); + if (unmasked_stransfer( + dst, dst_stride, src, src_stride, + subloopsize, src_itemsize, unmasked_transferdata) < 0) { + return -1; + } dst += subloopsize * dst_stride; src += subloopsize * src_stride; N -= subloopsize; } + return 0; } -static void _strided_masked_wrapper_transfer_function( +static int +_strided_masked_wrapper_transfer_function( char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_bool *mask, npy_intp mask_stride, @@ -3020,18 +3089,22 @@ static void _strided_masked_wrapper_transfer_function( /* Process unmasked values */ mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N, &subloopsize, 0); - unmasked_stransfer(dst, dst_stride, src, src_stride, - subloopsize, src_itemsize, unmasked_transferdata); + if (unmasked_stransfer( + dst, dst_stride, src, src_stride, + subloopsize, src_itemsize, unmasked_transferdata) < 0) { + return -1; + } dst += subloopsize * dst_stride; src += subloopsize * src_stride; N -= subloopsize; } + return 0; } /************************* DEST BOOL SETONE *******************************/ -static void +static int _null_to_strided_set_bool_one(char *dst, npy_intp dst_stride, char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), @@ -3046,9 +3119,10 @@ _null_to_strided_set_bool_one(char *dst, dst += dst_stride; --N; } + return 0; } -static void +static int _null_to_contig_set_bool_one(char *dst, npy_intp NPY_UNUSED(dst_stride), char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), @@ -3058,6 +3132,7 @@ _null_to_contig_set_bool_one(char *dst, /* bool type is one byte, so can just use the char */ memset(dst, 1, N); + return 0; } /* Only for the bool type, sets the destination to 1 */ @@ -3101,7 +3176,7 @@ static NpyAuxData *_dst_memset_zero_data_clone(NpyAuxData *data) return (NpyAuxData *)newdata; } -static void +static int _null_to_strided_memset_zero(char *dst, npy_intp dst_stride, char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), @@ -3116,9 +3191,10 @@ _null_to_strided_memset_zero(char *dst, dst += dst_stride; --N; } + return 0; } -static void +static int _null_to_contig_memset_zero(char *dst, npy_intp dst_stride, char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), @@ -3129,9 +3205,10 @@ _null_to_contig_memset_zero(char *dst, npy_intp dst_itemsize = d->dst_itemsize; memset(dst, 0, N*dst_itemsize); + return 0; } -static void +static int _null_to_strided_reference_setzero(char *dst, npy_intp dst_stride, char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), @@ -3143,17 +3220,15 @@ _null_to_strided_reference_setzero(char *dst, while (N > 0) { NPY_COPY_PYOBJECT_PTR(&dst_ref, dst); - /* Release the reference in dst */ + /* Release the reference in dst and set it to NULL */ NPY_DT_DBG_REFTRACE("dec dest ref (to set zero)", dst_ref); Py_XDECREF(dst_ref); - - /* Set it to zero */ - dst_ref = NULL; - NPY_COPY_PYOBJECT_PTR(dst, &dst_ref); + memset(dst, 0, sizeof(PyObject *)); dst += dst_stride; --N; } + return 0; } NPY_NO_EXPORT int @@ -3250,7 +3325,7 @@ get_setdstzero_transfer_function(int aligned, return NPY_SUCCEED; } -static void +static int _dec_src_ref_nop(char *NPY_UNUSED(dst), npy_intp NPY_UNUSED(dst_stride), char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), @@ -3259,9 +3334,10 @@ _dec_src_ref_nop(char *NPY_UNUSED(dst), NpyAuxData *NPY_UNUSED(data)) { /* NOP */ + return 0; } -static void +static int _strided_to_null_dec_src_ref_reference(char *NPY_UNUSED(dst), npy_intp NPY_UNUSED(dst_stride), char *src, npy_intp src_stride, @@ -3271,15 +3347,16 @@ _strided_to_null_dec_src_ref_reference(char *NPY_UNUSED(dst), { PyObject *src_ref = NULL; while (N > 0) { - NPY_COPY_PYOBJECT_PTR(&src_ref, src); - - /* Release the reference in src */ + /* Release the reference in src and set it to NULL */ NPY_DT_DBG_REFTRACE("dec src ref (null dst)", src_ref); + NPY_COPY_PYOBJECT_PTR(&src_ref, src); Py_XDECREF(src_ref); + memset(src, 0, sizeof(PyObject *)); src += src_stride; --N; } + return 0; } diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src index d234c366c..9dc802508 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src @@ -110,7 +110,7 @@ * if not it can decrease performance * tested to improve performance on intel xeon 5x/7x, core2duo, amd phenom x4 */ -static void +static int #if @is_aligned@ && @is_swap@ == 0 && @elsize@ <= NPY_SIZEOF_INTP NPY_GCC_UNROLL_LOOPS #endif @@ -171,6 +171,7 @@ static void --N; } + return 0; } #endif @@ -182,7 +183,7 @@ static void * but it profits from vectorization enabled with -O3 */ #if (@src_contig@ == 0) && @is_aligned@ -static NPY_GCC_OPT_3 void +static NPY_GCC_OPT_3 int @prefix@_@oper@_size@elsize@_srcstride0(char *dst, npy_intp dst_stride, char *src, npy_intp NPY_UNUSED(src_stride), @@ -197,7 +198,7 @@ static NPY_GCC_OPT_3 void npy_uint64 temp0, temp1; #endif if (N == 0) { - return; + return 0; } #if @is_aligned@ && @elsize@ != 16 /* sanity check */ @@ -238,6 +239,7 @@ static NPY_GCC_OPT_3 void --N; } #endif/* @elsize == 1 && @dst_contig@ -- else */ + return 0; } #endif/* (@src_contig@ == 0) && @is_aligned@ */ @@ -247,7 +249,7 @@ static NPY_GCC_OPT_3 void /**end repeat1**/ /**end repeat**/ -static void +static int _strided_to_strided(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -259,9 +261,10 @@ _strided_to_strided(char *dst, npy_intp dst_stride, src += src_stride; --N; } + return 0; } -static void +static int _swap_strided_to_strided(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -284,9 +287,10 @@ _swap_strided_to_strided(char *dst, npy_intp dst_stride, src += src_stride; --N; } + return 0; } -static void +static int _swap_pair_strided_to_strided(char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, npy_intp N, npy_intp src_itemsize, @@ -319,15 +323,17 @@ _swap_pair_strided_to_strided(char *dst, npy_intp dst_stride, src += src_stride; --N; } + return 0; } -static void +static int _contig_to_contig(char *dst, npy_intp NPY_UNUSED(dst_stride), char *src, npy_intp NPY_UNUSED(src_stride), npy_intp N, npy_intp src_itemsize, NpyAuxData *NPY_UNUSED(data)) { memmove(dst, src, src_itemsize*N); + return 0; } @@ -787,7 +793,7 @@ NPY_NO_EXPORT PyArray_StridedUnaryOp * #endif -static NPY_GCC_OPT_3 void +static NPY_GCC_OPT_3 int @prefix@_cast_@name1@_to_@name2@( char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, @@ -873,6 +879,7 @@ static NPY_GCC_OPT_3 void src += src_stride; #endif } + return 0; } #undef _CONVERT_FN @@ -989,10 +996,14 @@ PyArray_TransferNDimToStrided(npy_intp ndim, src_stride0 = src_strides[0]; N = shape0 - coord0; if (N >= count) { - stransfer(dst, dst_stride, src, src_stride0, count, src_itemsize, data); - return 0; + return stransfer(dst, dst_stride, src, src_stride0, + count, src_itemsize, data); + } + int res = stransfer(dst, dst_stride, src, src_stride0, + N, src_itemsize, data); + if (res < 0) { + return -1; } - stransfer(dst, dst_stride, src, src_stride0, N, src_itemsize, data); count -= N; /* If it's 1-dimensional, there's no more to copy */ @@ -1012,13 +1023,15 @@ PyArray_TransferNDimToStrided(npy_intp ndim, N = shape0*M; for (i = 0; i < M; ++i) { if (shape0 >= count) { - stransfer(dst, dst_stride, src, src_stride0, - count, src_itemsize, data); - return 0; + return stransfer(dst, dst_stride, src, src_stride0, + count, src_itemsize, data); } else { - stransfer(dst, dst_stride, src, src_stride0, - shape0, src_itemsize, data); + res = stransfer(dst, dst_stride, src, src_stride0, + shape0, src_itemsize, data); + if (res < 0) { + return -1; + } } count -= shape0; src += src_stride1; @@ -1073,13 +1086,15 @@ PyArray_TransferNDimToStrided(npy_intp ndim, /* A loop for dimensions 0 and 1 */ for (i = 0; i < shape1; ++i) { if (shape0 >= count) { - stransfer(dst, dst_stride, src, src_stride0, - count, src_itemsize, data); - return 0; + return stransfer(dst, dst_stride, src, src_stride0, + count, src_itemsize, data); } else { - stransfer(dst, dst_stride, src, src_stride0, - shape0, src_itemsize, data); + res = stransfer(dst, dst_stride, src, src_stride0, + shape0, src_itemsize, data); + if (res < 0) { + return -1; + } } count -= shape0; src += src_stride1; @@ -1108,10 +1123,14 @@ PyArray_TransferStridedToNDim(npy_intp ndim, dst_stride0 = dst_strides[0]; N = shape0 - coord0; if (N >= count) { - stransfer(dst, dst_stride0, src, src_stride, count, src_itemsize, data); - return 0; + return stransfer(dst, dst_stride0, src, src_stride, + count, src_itemsize, data); + } + int res = stransfer(dst, dst_stride0, src, src_stride, + N, src_itemsize, data); + if (res < 0) { + return -1; } - stransfer(dst, dst_stride0, src, src_stride, N, src_itemsize, data); count -= N; /* If it's 1-dimensional, there's no more to copy */ @@ -1131,13 +1150,15 @@ PyArray_TransferStridedToNDim(npy_intp ndim, N = shape0*M; for (i = 0; i < M; ++i) { if (shape0 >= count) { - stransfer(dst, dst_stride0, src, src_stride, - count, src_itemsize, data); - return 0; + return stransfer(dst, dst_stride0, src, src_stride, + count, src_itemsize, data); } else { - stransfer(dst, dst_stride0, src, src_stride, - shape0, src_itemsize, data); + res = stransfer(dst, dst_stride0, src, src_stride, + shape0, src_itemsize, data); + if (res < 0) { + return -1; + } } count -= shape0; dst += dst_stride1; @@ -1192,13 +1213,15 @@ PyArray_TransferStridedToNDim(npy_intp ndim, /* A loop for dimensions 0 and 1 */ for (i = 0; i < shape1; ++i) { if (shape0 >= count) { - stransfer(dst, dst_stride0, src, src_stride, - count, src_itemsize, data); - return 0; + return stransfer(dst, dst_stride0, src, src_stride, + count, src_itemsize, data); } else { - stransfer(dst, dst_stride0, src, src_stride, - shape0, src_itemsize, data); + res = stransfer(dst, dst_stride0, src, src_stride, + shape0, src_itemsize, data); + if (res < 0) { + return -1; + } } count -= shape0; dst += dst_stride1; @@ -1228,16 +1251,18 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, dst_stride0 = dst_strides[0]; N = shape0 - coord0; if (N >= count) { - stransfer(dst, dst_stride0, - src, src_stride, - mask, mask_stride, - count, src_itemsize, data); - return 0; - } - stransfer(dst, dst_stride0, - src, src_stride, + return stransfer( + dst, dst_stride0, src, src_stride, mask, mask_stride, - N, src_itemsize, data); + count, src_itemsize, data); + } + int res = stransfer( + dst, dst_stride0, src, src_stride, + mask, mask_stride, + N, src_itemsize, data); + if (res < 0) { + return -1; + } count -= N; /* If it's 1-dimensional, there's no more to copy */ @@ -1258,17 +1283,19 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, N = shape0*M; for (i = 0; i < M; ++i) { if (shape0 >= count) { - stransfer(dst, dst_stride0, - src, src_stride, - mask, mask_stride, - count, src_itemsize, data); - return 0; + return stransfer( + dst, dst_stride0, src, src_stride, + mask, mask_stride, + count, src_itemsize, data); } else { - stransfer(dst, dst_stride0, - src, src_stride, - mask, mask_stride, - shape0, src_itemsize, data); + int res = stransfer( + dst, dst_stride0, src, src_stride, + mask, mask_stride, + N, src_itemsize, data); + if (res < 0) { + return -1; + } } count -= shape0; dst += dst_stride1; @@ -1324,17 +1351,19 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, /* A loop for dimensions 0 and 1 */ for (i = 0; i < shape1; ++i) { if (shape0 >= count) { - stransfer(dst, dst_stride0, - src, src_stride, - mask, mask_stride, - count, src_itemsize, data); - return 0; + return stransfer( + dst, dst_stride0, src, src_stride, + mask, mask_stride, + count, src_itemsize, data); } else { - stransfer(dst, dst_stride0, - src, src_stride, - mask, mask_stride, - shape0, src_itemsize, data); + res = stransfer( + dst, dst_stride0, src, src_stride, + mask, mask_stride, + shape0, src_itemsize, data); + if (res < 0) { + return -1; + } } count -= shape0; dst += dst_stride1; @@ -1760,13 +1789,23 @@ mapiter_@name@(PyArrayMapIterObject *mit) do { #if @isget@ - stransfer(subspace_ptrs[1], subspace_strides[1], - subspace_ptrs[0], subspace_strides[0], - *counter, src_itemsize, transferdata); + if (NPY_UNLIKELY(stransfer( + subspace_ptrs[1], subspace_strides[1], + subspace_ptrs[0], subspace_strides[0], + *counter, src_itemsize, transferdata) < 0)) { + NPY_END_THREADS; + NPY_AUXDATA_FREE(transferdata); + return -1; + } #else - stransfer(subspace_ptrs[0], subspace_strides[0], - subspace_ptrs[1], subspace_strides[1], - *counter, src_itemsize, transferdata); + if (NPY_UNLIKELY(stransfer( + subspace_ptrs[0], subspace_strides[0], + subspace_ptrs[1], subspace_strides[1], + *counter, src_itemsize, transferdata) < 0)) { + NPY_END_THREADS; + NPY_AUXDATA_FREE(transferdata); + return -1; + } #endif } while (mit->subspace_next(mit->subspace_iter)); diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index 1f2bec8b1..db15ff1d5 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1091,6 +1091,7 @@ array_boolean_subscript(PyArrayObject *self, self_stride = innerstrides[0]; bmask_stride = innerstrides[1]; + int res = 0; do { innersize = *NpyIter_GetInnerLoopSizePtr(iter); self_data = dataptrs[0]; @@ -1105,8 +1106,11 @@ array_boolean_subscript(PyArrayObject *self, /* Process unmasked values */ bmask_data = npy_memchr(bmask_data, 0, bmask_stride, innersize, &subloopsize, 0); - stransfer(ret_data, itemsize, self_data, self_stride, - subloopsize, itemsize, transferdata); + res = stransfer(ret_data, itemsize, self_data, self_stride, + subloopsize, itemsize, transferdata); + if (res < 0) { + break; + } innersize -= subloopsize; self_data += subloopsize * self_stride; ret_data += subloopsize * itemsize; @@ -1115,8 +1119,15 @@ array_boolean_subscript(PyArrayObject *self, NPY_END_THREADS; - NpyIter_Deallocate(iter); + if (!NpyIter_Deallocate(iter)) { + res = -1; + } NPY_AUXDATA_FREE(transferdata); + if (res < 0) { + /* Should be practically impossible, since there is no cast */ + Py_DECREF(ret); + return NULL; + } } if (!PyArray_CheckExact(self)) { @@ -1209,6 +1220,7 @@ array_assign_boolean_subscript(PyArrayObject *self, v_data = PyArray_DATA(v); /* Create an iterator for the data */ + int res = 0; if (size > 0) { NpyIter *iter; PyArrayObject *op[2] = {self, bmask}; @@ -1253,7 +1265,7 @@ array_assign_boolean_subscript(PyArrayObject *self, /* Get a dtype transfer function */ NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); if (PyArray_GetDTypeTransferFunction( - IsUintAligned(self) && IsAligned(self) && + IsUintAligned(self) && IsAligned(self) && IsUintAligned(v) && IsAligned(v), v_stride, fixed_strides[0], PyArray_DESCR(v), PyArray_DESCR(self), @@ -1282,8 +1294,11 @@ array_assign_boolean_subscript(PyArrayObject *self, /* Process unmasked values */ bmask_data = npy_memchr(bmask_data, 0, bmask_stride, innersize, &subloopsize, 0); - stransfer(self_data, self_stride, v_data, v_stride, - subloopsize, src_itemsize, transferdata); + res = stransfer(self_data, self_stride, v_data, v_stride, + subloopsize, src_itemsize, transferdata); + if (res < 0) { + break; + } innersize -= subloopsize; self_data += subloopsize * self_stride; v_data += subloopsize * v_stride; @@ -1295,22 +1310,12 @@ array_assign_boolean_subscript(PyArrayObject *self, } NPY_AUXDATA_FREE(transferdata); - NpyIter_Deallocate(iter); - } - - if (needs_api) { - /* - * FIXME?: most assignment operations stop after the first occurrence - * of an error. Boolean does not currently, but should at least - * report the error. (This is only relevant for things like str->int - * casts which call into python) - */ - if (PyErr_Occurred()) { - return -1; + if (!NpyIter_Deallocate(iter)) { + res = -1; } } - return 0; + return res; } diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index a5b5e5c51..059f2c437 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -229,13 +229,22 @@ NpyIter_EnableExternalLoop(NpyIter *iter) return NpyIter_Reset(iter, NULL); } + +static char *_reset_cast_error = ( + "Iterator reset failed due to a casting failure. " + "This error is set as a Python error."); + /*NUMPY_API * Resets the iterator to its initial state * + * The use of errmsg is discouraged, it cannot be guaranteed that the GIL + * will not be grabbed on casting errors even when this is passed. + * * If errmsg is non-NULL, it should point to a variable which will * receive the error message, and no Python exception will be set. * This is so that the function can be called from code not holding - * the GIL. + * the GIL. Note that cast errors may still lead to the GIL being + * grabbed temporarily. */ NPY_NO_EXPORT int NpyIter_Reset(NpyIter *iter, char **errmsg) @@ -250,6 +259,9 @@ NpyIter_Reset(NpyIter *iter, char **errmsg) /* If buffer allocation was delayed, do it now */ if (itflags&NPY_ITFLAG_DELAYBUF) { if (!npyiter_allocate_buffers(iter, errmsg)) { + if (errmsg != NULL) { + *errmsg = _reset_cast_error; + } return NPY_FAIL; } NIT_ITFLAGS(iter) &= ~NPY_ITFLAG_DELAYBUF; @@ -257,7 +269,7 @@ NpyIter_Reset(NpyIter *iter, char **errmsg) else { /* * If the iterindex is already right, no need to - * do anything + * do anything (and no cast error has previously occurred). */ bufferdata = NIT_BUFFERDATA(iter); if (NIT_ITERINDEX(iter) == NIT_ITERSTART(iter) && @@ -265,9 +277,12 @@ NpyIter_Reset(NpyIter *iter, char **errmsg) NBF_SIZE(bufferdata) > 0) { return NPY_SUCCEED; } - - /* Copy any data from the buffers back to the arrays */ - npyiter_copy_from_buffers(iter); + if (npyiter_copy_from_buffers(iter) < 0) { + if (errmsg != NULL) { + *errmsg = _reset_cast_error; + } + return NPY_FAIL; + } } } @@ -275,7 +290,12 @@ NpyIter_Reset(NpyIter *iter, char **errmsg) if (itflags&NPY_ITFLAG_BUFFER) { /* Prepare the next buffers and set iterend/size */ - npyiter_copy_to_buffers(iter, NULL); + if (npyiter_copy_to_buffers(iter, NULL) < 0) { + if (errmsg != NULL) { + *errmsg = _reset_cast_error; + } + return NPY_FAIL; + } } return NPY_SUCCEED; @@ -288,7 +308,8 @@ NpyIter_Reset(NpyIter *iter, char **errmsg) * If errmsg is non-NULL, it should point to a variable which will * receive the error message, and no Python exception will be set. * This is so that the function can be called from code not holding - * the GIL. + * the GIL. Note that cast errors may still lead to the GIL being + * grabbed temporarily. */ NPY_NO_EXPORT int NpyIter_ResetBasePointers(NpyIter *iter, char **baseptrs, char **errmsg) @@ -309,8 +330,12 @@ NpyIter_ResetBasePointers(NpyIter *iter, char **baseptrs, char **errmsg) NIT_ITFLAGS(iter) &= ~NPY_ITFLAG_DELAYBUF; } else { - /* Copy any data from the buffers back to the arrays */ - npyiter_copy_from_buffers(iter); + if (npyiter_copy_from_buffers(iter) < 0) { + if (errmsg != NULL) { + *errmsg = _reset_cast_error; + } + return NPY_FAIL; + } } } @@ -323,7 +348,12 @@ NpyIter_ResetBasePointers(NpyIter *iter, char **baseptrs, char **errmsg) if (itflags&NPY_ITFLAG_BUFFER) { /* Prepare the next buffers and set iterend/size */ - npyiter_copy_to_buffers(iter, NULL); + if (npyiter_copy_to_buffers(iter, NULL) < 0) { + if (errmsg != NULL) { + *errmsg = _reset_cast_error; + } + return NPY_FAIL; + } } return NPY_SUCCEED; @@ -335,7 +365,8 @@ NpyIter_ResetBasePointers(NpyIter *iter, char **baseptrs, char **errmsg) * If errmsg is non-NULL, it should point to a variable which will * receive the error message, and no Python exception will be set. * This is so that the function can be called from code not holding - * the GIL. + * the GIL. Note that cast errors may still lead to the GIL being + * grabbed temporarily. */ NPY_NO_EXPORT int NpyIter_ResetToIterIndexRange(NpyIter *iter, @@ -633,12 +664,16 @@ NpyIter_GotoIterIndex(NpyIter *iter, npy_intp iterindex) /* Start the buffer at the provided iterindex */ else { /* Write back to the arrays */ - npyiter_copy_from_buffers(iter); + if (npyiter_copy_from_buffers(iter) < 0) { + return NPY_FAIL; + } npyiter_goto_iterindex(iter, iterindex); /* Prepare the next buffers and set iterend/size */ - npyiter_copy_to_buffers(iter, NULL); + if (npyiter_copy_to_buffers(iter, NULL) < 0) { + return NPY_FAIL; + } } } else { @@ -1376,6 +1411,7 @@ NpyIter_GetInnerLoopSizePtr(NpyIter *iter) } } + /*NUMPY_API * For debugging */ @@ -1828,7 +1864,7 @@ npyiter_goto_iterindex(NpyIter *iter, npy_intp iterindex) * their data needs to be written back to the arrays. The multi-index * must be positioned for the beginning of the buffer. */ -NPY_NO_EXPORT void +NPY_NO_EXPORT int npyiter_copy_from_buffers(NpyIter *iter) { npy_uint32 itflags = NIT_ITFLAGS(iter); @@ -1861,7 +1897,7 @@ npyiter_copy_from_buffers(NpyIter *iter) /* If we're past the end, nothing to copy */ if (NBF_SIZE(bufferdata) == 0) { - return; + return 0; } NPY_IT_DBG_PRINT("Iterator: Copying buffers to outputs\n"); @@ -1968,7 +2004,7 @@ npyiter_copy_from_buffers(NpyIter *iter) maskptr = (npy_bool *)ad_ptrs[maskop]; } - PyArray_TransferMaskedStridedToNDim(ndim_transfer, + if (PyArray_TransferMaskedStridedToNDim(ndim_transfer, ad_ptrs[iop], dst_strides, axisdata_incr, buffer, src_stride, maskptr, strides[maskop], @@ -1976,18 +2012,22 @@ npyiter_copy_from_buffers(NpyIter *iter) dst_shape, axisdata_incr, op_transfersize, dtypes[iop]->elsize, (PyArray_MaskedStridedUnaryOp *)stransfer, - transferdata); + transferdata) < 0) { + return -1; + } } /* Regular operand */ else { - PyArray_TransferStridedToNDim(ndim_transfer, + if (PyArray_TransferStridedToNDim(ndim_transfer, ad_ptrs[iop], dst_strides, axisdata_incr, buffer, src_stride, dst_coords, axisdata_incr, dst_shape, axisdata_incr, op_transfersize, dtypes[iop]->elsize, stransfer, - transferdata); + transferdata) < 0) { + return -1; + } } } /* If there's no copy back, we may have to decrement refs. In @@ -2002,9 +2042,13 @@ npyiter_copy_from_buffers(NpyIter *iter) NPY_IT_DBG_PRINT1("Iterator: Freeing refs and zeroing buffer " "of operand %d\n", (int)iop); /* Decrement refs */ - stransfer(NULL, 0, buffer, dtypes[iop]->elsize, - transfersize, dtypes[iop]->elsize, - transferdata); + if (stransfer(NULL, 0, buffer, dtypes[iop]->elsize, + transfersize, dtypes[iop]->elsize, + transferdata) < 0) { + /* Since this should only decrement, it should never error */ + assert(0); + return -1; + } /* * Zero out the memory for safety. For instance, * if during iteration some Python code copied an @@ -2016,6 +2060,7 @@ npyiter_copy_from_buffers(NpyIter *iter) } NPY_IT_DBG_PRINT("Iterator: Finished copying buffers to outputs\n"); + return 0; } /* @@ -2023,7 +2068,7 @@ npyiter_copy_from_buffers(NpyIter *iter) * for the start of a buffer. It decides which operands need a buffer, * and copies the data into the buffers. */ -NPY_NO_EXPORT void +NPY_NO_EXPORT int npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) { npy_uint32 itflags = NIT_ITFLAGS(iter); @@ -2142,7 +2187,7 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) NBF_BUFITEREND(bufferdata) = iterindex + reduce_innersize; if (reduce_innersize == 0) { NBF_REDUCE_OUTERSIZE(bufferdata) = 0; - return; + return 0; } else { NBF_REDUCE_OUTERSIZE(bufferdata) = transfersize/reduce_innersize; @@ -2508,14 +2553,15 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) "buffer (%d items)\n", (int)iop, (int)op_transfersize); - PyArray_TransferNDimToStrided(ndim_transfer, - ptrs[iop], dst_stride, - ad_ptrs[iop], src_strides, axisdata_incr, - src_coords, axisdata_incr, - src_shape, axisdata_incr, - op_transfersize, src_itemsize, - stransfer, - transferdata); + if (PyArray_TransferNDimToStrided( + ndim_transfer, ptrs[iop], dst_stride, + ad_ptrs[iop], src_strides, axisdata_incr, + src_coords, axisdata_incr, + src_shape, axisdata_incr, + op_transfersize, src_itemsize, + stransfer, transferdata) < 0) { + return -1; + } } } else if (ptrs[iop] == buffers[iop]) { @@ -2551,8 +2597,80 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) NPY_IT_DBG_PRINT1("Iterator: Finished copying inputs to buffers " "(buffered size is %d)\n", (int)NBF_SIZE(bufferdata)); + return 0; } + +/** + * This function clears any references still held by the buffers and should + * only be used to discard buffers if an error occurred. + * + * @param iter Iterator + */ +NPY_NO_EXPORT void +npyiter_clear_buffers(NpyIter *iter) +{ + int nop = iter->nop; + NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter); + + if (NBF_SIZE(bufferdata) == 0) { + /* if the buffers are empty already, there is nothing to do */ + return; + } + + if (!(NIT_ITFLAGS(iter) & NPY_ITFLAG_NEEDSAPI)) { + /* Buffers do not require clearing, but should not be copied back */ + NBF_SIZE(bufferdata) = 0; + return; + } + + /* + * The iterator may be using a dtype with references, which always + * requires the API. In that case, further cleanup may be necessary. + * + * TODO: At this time, we assume that a dtype having references + * implies the need to hold the GIL at all times. In theory + * we could broaden this definition for a new + * `PyArray_Item_XDECREF` API and the assumption may become + * incorrect. + */ + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + + /* Cleanup any buffers with references */ + char **buffers = NBF_BUFFERS(bufferdata); + PyArray_Descr **dtypes = NIT_DTYPES(iter); + for (int iop = 0; iop < nop; ++iop, ++buffers) { + /* + * We may want to find a better way to do this, on the other hand, + * this cleanup seems rare and fairly special. A dtype using + * references (right now only us) must always keep the buffer in + * a well defined state (either NULL or owning the reference). + * Only we implement cleanup + */ + if (!PyDataType_REFCHK(dtypes[iop])) { + continue; + } + if (*buffers == 0) { + continue; + } + int itemsize = dtypes[iop]->elsize; + for (npy_intp i = 0; i < NBF_SIZE(bufferdata); i++) { + /* + * See above comment, if this API is expanded the GIL assumption + * could become incorrect. + */ + PyArray_Item_XDECREF(*buffers + (itemsize * i), dtypes[iop]); + } + /* Clear out the buffer just to be sure */ + memset(*buffers, 0, NBF_SIZE(bufferdata) * itemsize); + } + /* Signal that the buffers are empty */ + NBF_SIZE(bufferdata) = 0; + PyErr_Restore(type, value, traceback); +} + + /* * This checks how much space can be buffered without encountering the * same value twice, or for operands whose innermost stride is zero, diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index 7da17eafe..a0dda4090 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -476,7 +476,10 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, } /* Prepare the next buffers and set iterend/size */ - npyiter_copy_to_buffers(iter, NULL); + if (npyiter_copy_to_buffers(iter, NULL) < 0) { + NpyIter_Deallocate(iter); + return NULL; + } } } @@ -642,21 +645,27 @@ NpyIter_Copy(NpyIter *iter) } /*NUMPY_API - * Deallocate an iterator + * Deallocate an iterator. + * + * To correctly work when an error is in progress, we have to check + * `PyErr_Occurred()`. This is necessary when buffers are not finalized + * or WritebackIfCopy is used. We could avoid that check by exposing a new + * function which is passed in whether or not a Python error is already set. */ NPY_NO_EXPORT int NpyIter_Deallocate(NpyIter *iter) { + int success = PyErr_Occurred() == NULL; + npy_uint32 itflags; /*int ndim = NIT_NDIM(iter);*/ int iop, nop; PyArray_Descr **dtype; PyArrayObject **object; npyiter_opitflags *op_itflags; - npy_bool resolve = 1; if (iter == NULL) { - return NPY_SUCCEED; + return success; } itflags = NIT_ITFLAGS(iter); @@ -667,13 +676,23 @@ NpyIter_Deallocate(NpyIter *iter) /* Deallocate any buffers and buffering data */ if (itflags & NPY_ITFLAG_BUFFER) { + /* Ensure no data is held by the buffers before they are cleared */ + if (success) { + if (npyiter_copy_from_buffers(iter) < 0) { + success = NPY_FAIL; + } + } + else { + npyiter_clear_buffers(iter); + } + NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter); char **buffers; NpyAuxData **transferdata; /* buffers */ buffers = NBF_BUFFERS(bufferdata); - for(iop = 0; iop < nop; ++iop, ++buffers) { + for (iop = 0; iop < nop; ++iop, ++buffers) { PyArray_free(*buffers); } /* read bufferdata */ @@ -694,12 +713,12 @@ NpyIter_Deallocate(NpyIter *iter) /* * Deallocate all the dtypes and objects that were iterated and resolve - * any writeback buffers created by the iterator + * any writeback buffers created by the iterator. */ - for(iop = 0; iop < nop; ++iop, ++dtype, ++object) { + for (iop = 0; iop < nop; ++iop, ++dtype, ++object) { if (op_itflags[iop] & NPY_OP_ITFLAG_HAS_WRITEBACK) { - if (resolve && PyArray_ResolveWritebackIfCopy(*object) < 0) { - resolve = 0; + if (success && PyArray_ResolveWritebackIfCopy(*object) < 0) { + success = 0; } else { PyArray_DiscardWritebackIfCopy(*object); @@ -711,12 +730,10 @@ NpyIter_Deallocate(NpyIter *iter) /* Deallocate the iterator memory */ PyObject_Free(iter); - if (resolve == 0) { - return NPY_FAIL; - } - return NPY_SUCCEED; + return success; } + /* Checks 'flags' for (C|F)_ORDER_INDEX, MULTI_INDEX, and EXTERNAL_LOOP, * setting the appropriate internal flags in 'itflags'. * diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index 1477c8631..bb483bb1f 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -342,10 +342,11 @@ NPY_NO_EXPORT int npyiter_allocate_buffers(NpyIter *iter, char **errmsg); NPY_NO_EXPORT void npyiter_goto_iterindex(NpyIter *iter, npy_intp iterindex); -NPY_NO_EXPORT void +NPY_NO_EXPORT int npyiter_copy_from_buffers(NpyIter *iter); -NPY_NO_EXPORT void +NPY_NO_EXPORT int npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs); - +NPY_NO_EXPORT void +npyiter_clear_buffers(NpyIter *iter); #endif diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 7f31a5096..1c68a4803 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -1268,6 +1268,10 @@ npyiter_iternext(NewNpyArrayIterObject *self) Py_RETURN_TRUE; } else { + if (PyErr_Occurred()) { + /* casting error, buffer cleanup will occur at reset or dealloc */ + return NULL; + } self->finished = 1; Py_RETURN_FALSE; } @@ -1483,6 +1487,10 @@ npyiter_next(NewNpyArrayIterObject *self) */ if (self->started) { if (!self->iternext(self->iter)) { + /* + * A casting error may be set here (or no error causing a + * StopIteration). Buffers may only be cleaned up later. + */ self->finished = 1; return NULL; } diff --git a/numpy/core/src/multiarray/nditer_templ.c.src b/numpy/core/src/multiarray/nditer_templ.c.src index 0f0d59972..05ce6ae75 100644 --- a/numpy/core/src/multiarray/nditer_templ.c.src +++ b/numpy/core/src/multiarray/nditer_templ.c.src @@ -249,7 +249,10 @@ npyiter_buffered_reduce_iternext_iters@tag_nop@(NpyIter *iter) memcpy(prev_dataptrs, NAD_PTRS(axisdata), NPY_SIZEOF_INTP*nop); /* Write back to the arrays */ - npyiter_copy_from_buffers(iter); + if (npyiter_copy_from_buffers(iter) < 0) { + npyiter_clear_buffers(iter); + return 0; + } /* Check if we're past the end */ if (NIT_ITERINDEX(iter) >= NIT_ITEREND(iter)) { @@ -262,7 +265,10 @@ npyiter_buffered_reduce_iternext_iters@tag_nop@(NpyIter *iter) } /* Prepare the next buffers and set iterend/size */ - npyiter_copy_to_buffers(iter, prev_dataptrs); + if (npyiter_copy_to_buffers(iter, prev_dataptrs) < 0) { + npyiter_clear_buffers(iter); + return 0; + } return 1; } @@ -303,7 +309,10 @@ npyiter_buffered_iternext(NpyIter *iter) } /* Write back to the arrays */ - npyiter_copy_from_buffers(iter); + if (npyiter_copy_from_buffers(iter) < 0) { + npyiter_clear_buffers(iter); + return 0; + } /* Check if we're past the end */ if (NIT_ITERINDEX(iter) >= NIT_ITEREND(iter)) { @@ -316,7 +325,10 @@ npyiter_buffered_iternext(NpyIter *iter) } /* Prepare the next buffers and set iterend/size */ - npyiter_copy_to_buffers(iter, NULL); + if (npyiter_copy_to_buffers(iter, NULL) < 0) { + npyiter_clear_buffers(iter); + return 0; + } return 1; } diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index 4037a4757..d0fb2f6ed 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -254,7 +254,6 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, } op_flags[2] = NPY_ITER_READONLY; } - /* Set up result array axes mapping, operand and wheremask use default */ int result_axes[NPY_MAXDIMS]; int *op_axes[3] = {result_axes, NULL, NULL}; @@ -363,7 +362,6 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, if (loop(iter, dataptr, strideptr, countptr, iternext, needs_api, skip_first_count, data) < 0) { - goto fail; } } @@ -379,7 +377,10 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, } Py_INCREF(result); - NpyIter_Deallocate(iter); + if (!NpyIter_Deallocate(iter)) { + Py_DECREF(result); + return NULL; + } return result; fail: diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 8f841c6fa..b47ccd291 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -1536,7 +1536,14 @@ iterator_loop(PyUFuncObject *ufunc, NPY_END_THREADS; } - return NpyIter_Deallocate(iter); + /* + * Currently `innerloop` may leave an error set, in this case + * NpyIter_Deallocate will always return an error as well. + */ + if (NpyIter_Deallocate(iter) == NPY_FAIL) { + return -1; + } + return 0; } /* @@ -3233,9 +3240,13 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, goto fail; } - /* Check whether any errors occurred during the loop */ + /* + * Check whether any errors occurred during the loop. The loops should + * indicate this in retval, but since the inner-loop currently does not + * report errors, this does not happen in all branches (at this time). + */ if (PyErr_Occurred() || - _check_ufunc_fperr(errormask, extobj, ufunc_name) < 0) { + _check_ufunc_fperr(errormask, extobj, ufunc_name) < 0) { retval = -1; goto fail; } @@ -3997,8 +4008,17 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, finish: Py_XDECREF(op_dtypes[0]); - NpyIter_Deallocate(iter); - NpyIter_Deallocate(iter_inner); + int res = 0; + if (!NpyIter_Deallocate(iter)) { + res = -1; + } + if (!NpyIter_Deallocate(iter_inner)) { + res = -1; + } + if (res < 0) { + Py_DECREF(out); + return NULL; + } return (PyObject *)out; @@ -4379,7 +4399,10 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, finish: Py_XDECREF(op_dtypes[0]); - NpyIter_Deallocate(iter); + if (!NpyIter_Deallocate(iter)) { + Py_DECREF(out); + return NULL; + } return (PyObject *)out; @@ -4388,7 +4411,6 @@ fail: Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); - return NULL; } -- cgit v1.2.1 From 101192cbb5eedd8da97237a73ef2754fb9c65ddf Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 11 Aug 2020 18:22:19 -0500 Subject: TST: Add tests for correct nditer buffer clearing on cast error --- numpy/core/tests/test_nditer.py | 65 +++++++++++++++++++++++++++++++++++++++++ numpy/core/tests/test_ufunc.py | 60 ++++++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index 7b3c3a40d..e10c7ad92 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -2880,3 +2880,68 @@ def test_warn_noclose(): casting='equiv', op_dtypes=[np.dtype('f4')]) del it assert len(sup.log) == 1 + + +@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") +@pytest.mark.parametrize(["in_dtype", "buf_dtype"], + [("i", "O"), ("O", "i"), # most simple cases + ("i,O", "O,O"), # structured partially only copying O + ("O,i", "i,O"), # structured casting to and from O + ]) +@pytest.mark.parametrize("steps", [1, 2, 3]) +def test_partial_iteration_cleanup(in_dtype, buf_dtype, steps): + value = 123 # relies on python cache (leak-check will still find it) + arr = np.full(int(np.BUFSIZE * 2.5), value).astype(in_dtype) + count = sys.getrefcount(value) + + it = np.nditer(arr, op_dtypes=[np.dtype(buf_dtype)], + flags=["buffered", "external_loop", "refs_ok"], casting="unsafe") + for step in range(steps): + # The iteration finishes in 3 steps, the first two are partial + next(it) + + # Note that resetting does not free references + del it + assert count == sys.getrefcount(value) + + # Repeat the test with `iternext` + it = np.nditer(arr, op_dtypes=[np.dtype(buf_dtype)], + flags=["buffered", "external_loop", "refs_ok"], casting="unsafe") + for step in range(steps): + it.iternext() + + del it # should ensure cleanup + assert count == sys.getrefcount(value) + + +@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") +@pytest.mark.parametrize(["in_dtype", "buf_dtype"], + [("O", "i"), # most simple cases + ("O,i", "i,O"), # structured casting to and from O + ]) +def test_partial_iteration_error(in_dtype, buf_dtype): + value = 123 # relies on python cache (leak-check will still find it) + arr = np.full(int(np.BUFSIZE * 2.5), value).astype(in_dtype) + if in_dtype == "O": + arr[int(np.BUFSIZE * 1.5)] = None + else: + arr[int(np.BUFSIZE * 1.5)]["f0"] = None + + count = sys.getrefcount(value) + + it = np.nditer(arr, op_dtypes=[np.dtype(buf_dtype)], + flags=["buffered", "external_loop", "refs_ok"], casting="unsafe") + with pytest.raises(TypeError): + # pytest.raises seems to have issues with the error originating + # in the for loop, so manually unravel: + next(it) + next(it) # raises TypeError + + # Repeat the test with `iternext` after resetting, the buffers should + # already be cleared from any references, so resetting is sufficient. + it.reset() + with pytest.raises(TypeError): + it.iternext() + it.iternext() + + assert count == sys.getrefcount(value) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 1305f4877..9eaa1a977 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1,5 +1,6 @@ import warnings import itertools +import sys import pytest @@ -11,7 +12,7 @@ import numpy.core._rational_tests as _rational_tests from numpy.testing import ( assert_, assert_equal, assert_raises, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_no_warnings, - assert_allclose, + assert_allclose, HAS_REFCOUNT, ) from numpy.compat import pickle @@ -2074,3 +2075,60 @@ def test_ufunc_warn_with_nan(ufunc): else: raise ValueError('ufunc with more than 2 inputs') + +@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") +def test_ufunc_casterrors(): + # Tests that casting errors are correctly reported and buffers are + # cleared. + # The following array can be added to itself as an object array, but + # the result cannot be cast to an integer output: + value = 123 # relies on python cache (leak-check will still find it) + arr = np.array([value] * int(np.BUFSIZE * 1.5) + + ["string"] + + [value] * int(1.5 * np.BUFSIZE), dtype=object) + out = np.ones(len(arr), dtype=np.intp) + + count = sys.getrefcount(value) + with pytest.raises(ValueError): + # Output casting failure: + np.add(arr, arr, out=out, casting="unsafe") + + assert count == sys.getrefcount(value) + # output is unchanged after the error, this shows that the iteration + # was aborted (this is not necessarily defined behaviour) + assert out[-1] == 1 + + with pytest.raises(ValueError): + # Input casting failure: + np.add(arr, arr, out=out, dtype=np.intp, casting="unsafe") + + assert count == sys.getrefcount(value) + # output is unchanged after the error, this shows that the iteration + # was aborted (this is not necessarily defined behaviour) + assert out[-1] == 1 + + +@pytest.mark.skipif(not HAS_REFCOUNT, reason="Python lacks refcounts") +@pytest.mark.parametrize("offset", + [0, np.BUFSIZE//2, int(1.5*np.BUFSIZE)]) +def test_reduce_casterrors(offset): + # Test reporting of casting errors in reductions, we test various + # offsets to where the casting error will occur, since these may occur + # at different places during the reduction procedure. For example + # the first item may be special. + value = 123 # relies on python cache (leak-check will still find it) + arr = np.array([value] * offset + + ["string"] + + [value] * int(1.5 * np.BUFSIZE), dtype=object) + out = np.array(-1, dtype=np.intp) + + count = sys.getrefcount(value) + with pytest.raises(ValueError): + # This is an unsafe cast, but we currently always allow that: + np.add.reduce(arr, dtype=np.intp, out=out) + assert count == sys.getrefcount(value) + # If an error occurred during casting, the operation is done at most until + # the error occurs (the result of which would be `value * offset`) and -1 + # if the error happened immediately. + # This does not define behaviour, the output is invalid and thus undefined + assert out[()] < value * offset -- cgit v1.2.1 From 0328a8c3d4b734be8a4be82ab85f1971093da3da Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 11 Aug 2020 19:13:05 -0500 Subject: DOC: Add release note for casting error changes --- .../upcoming_changes/17029.compatibility.rst | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 doc/release/upcoming_changes/17029.compatibility.rst diff --git a/doc/release/upcoming_changes/17029.compatibility.rst b/doc/release/upcoming_changes/17029.compatibility.rst new file mode 100644 index 000000000..351b228a8 --- /dev/null +++ b/doc/release/upcoming_changes/17029.compatibility.rst @@ -0,0 +1,23 @@ +Casting errors interrupt Iteration +---------------------------------- +NumPy sometimes has issues reporting errors during casting +correctly. This is rare, since the vast majority of casts +will never result in errors. +However, in some cases of casting errors less of the result +may be written back. The state of arrays involved in errors +should *always* be considered undefined. +Possible changes occur for: + +* Universal functions when casting is unsafe (rare) +* Advanced index assignments +* The lowlevel `numpy.nditer` API. + +For users of the ``NpyIter`` C-API such cast errors will now +cause the `iternext()` function to return 0 and thus abort +iteration. +Currently, there is no API to detect such an error directly. +It is necessary to check ``PyErr_Occurred()``, which +may be problematic in combination with ``NpyIter_Reset``. +These issues always existed, but new API could be added +if required by users. + -- cgit v1.2.1 From a2b9c2d5b6637b040917c0a2ef393dae83f09ee3 Mon Sep 17 00:00:00 2001 From: peterbell10 Date: Wed, 12 Aug 2020 07:36:07 +0100 Subject: API, BUG: Raise error on complex input to i0 (#17062) * BUG, API: Raise error on complex input to np.i0 --- numpy/lib/function_base.py | 23 ++++++++++------------- numpy/lib/tests/test_function_base.py | 9 +++++++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index cd8862c94..b530f0aa1 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -3193,25 +3193,18 @@ def i0(x): """ Modified Bessel function of the first kind, order 0. - Usually denoted :math:`I_0`. This function does broadcast, but will *not* - "up-cast" int dtype arguments unless accompanied by at least one float or - complex dtype argument (see Raises below). + Usually denoted :math:`I_0`. Parameters ---------- - x : array_like, dtype float or complex + x : array_like of float Argument of the Bessel function. Returns ------- - out : ndarray, shape = x.shape, dtype = x.dtype + out : ndarray, shape = x.shape, dtype = float The modified Bessel function evaluated at each of the elements of `x`. - Raises - ------ - TypeError: array cannot be safely cast to required type - If argument consists exclusively of int dtypes. - See Also -------- scipy.special.i0, scipy.special.iv, scipy.special.ive @@ -3241,12 +3234,16 @@ def i0(x): Examples -------- >>> np.i0(0.) - array(1.0) # may vary - >>> np.i0([0., 1. + 2j]) - array([ 1.00000000+0.j , 0.18785373+0.64616944j]) # may vary + array(1.0) + >>> np.i0([0, 1, 2, 3]) + array([1. , 1.26606588, 2.2795853 , 4.88079259]) """ x = np.asanyarray(x) + if x.dtype.kind == 'c': + raise TypeError("i0 not supported for complex values") + if x.dtype.kind != 'f': + x = x.astype(float) x = np.abs(x) return piecewise(x, [x <= 8.0], [_i0_1, _i0_2]) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 89c1a2d9b..635fe1432 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -2111,8 +2111,9 @@ class Test_I0: i0(0.5), np.array(1.0634833707413234)) - A = np.array([0.49842636, 0.6969809, 0.22011976, 0.0155549]) - expected = np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049]) + # need at least one test above 8, as the implementation is piecewise + A = np.array([0.49842636, 0.6969809, 0.22011976, 0.0155549, 10.0]) + expected = np.array([1.06307822, 1.12518299, 1.01214991, 1.00006049, 2815.71662847]) assert_almost_equal(i0(A), expected) assert_almost_equal(i0(-A), expected) @@ -2149,6 +2150,10 @@ class Test_I0: assert_array_equal(exp, res) + def test_complex(self): + a = np.array([0, 1 + 2j]) + with pytest.raises(TypeError, match="i0 not supported for complex values"): + res = i0(a) class TestKaiser: -- cgit v1.2.1 From 2b54640ad50bbbfecfb4c9cc8c9dd9c161f6a52e Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Wed, 12 Aug 2020 19:55:50 +0200 Subject: MAINT: Remove _EXTRAFLAGS variable (#17050) Delete unused _EXTRAFLAGS variable throughout file --- numpy/distutils/fcompiler/gnu.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index caa08549e..7004b5d80 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -23,13 +23,6 @@ def is_win64(): return sys.platform == "win32" and platform.architecture()[0] == "64bit" -if is_win64(): - #_EXTRAFLAGS = ["-fno-leading-underscore"] - _EXTRAFLAGS = [] -else: - _EXTRAFLAGS = [] - - class GnuFCompiler(FCompiler): compiler_type = 'gnu' compiler_aliases = ('g77', ) @@ -297,11 +290,11 @@ class Gnu95FCompiler(GnuFCompiler): executables = { 'version_cmd' : ["", "-dumpversion"], 'compiler_f77' : [None, "-Wall", "-g", "-ffixed-form", - "-fno-second-underscore"] + _EXTRAFLAGS, + "-fno-second-underscore"], 'compiler_f90' : [None, "-Wall", "-g", - "-fno-second-underscore"] + _EXTRAFLAGS, + "-fno-second-underscore"], 'compiler_fix' : [None, "-Wall", "-g","-ffixed-form", - "-fno-second-underscore"] + _EXTRAFLAGS, + "-fno-second-underscore"], 'linker_so' : ["", "-Wall", "-g"], 'archiver' : ["ar", "-cr"], 'ranlib' : ["ranlib"], -- cgit v1.2.1 From f89486c781634ad53e7107b35fc3899822734908 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 12 Aug 2020 16:55:37 -0500 Subject: Update doc/release/upcoming_changes/17029.compatibility.rst --- doc/release/upcoming_changes/17029.compatibility.rst | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/doc/release/upcoming_changes/17029.compatibility.rst b/doc/release/upcoming_changes/17029.compatibility.rst index 351b228a8..69069ce18 100644 --- a/doc/release/upcoming_changes/17029.compatibility.rst +++ b/doc/release/upcoming_changes/17029.compatibility.rst @@ -1,17 +1,9 @@ Casting errors interrupt Iteration ---------------------------------- -NumPy sometimes has issues reporting errors during casting -correctly. This is rare, since the vast majority of casts -will never result in errors. -However, in some cases of casting errors less of the result -may be written back. The state of arrays involved in errors -should *always* be considered undefined. -Possible changes occur for: - -* Universal functions when casting is unsafe (rare) -* Advanced index assignments -* The lowlevel `numpy.nditer` API. - +When iterating while casting values, an error may stop the iteration +earlier than before. In any case, a failed casting operation always +returned undefined, partial results. Those may now be even more +undefined and partial. For users of the ``NpyIter`` C-API such cast errors will now cause the `iternext()` function to return 0 and thus abort iteration. @@ -20,4 +12,3 @@ It is necessary to check ``PyErr_Occurred()``, which may be problematic in combination with ``NpyIter_Reset``. These issues always existed, but new API could be added if required by users. - -- cgit v1.2.1 From b4fd7a79bc55f17f301d219492528e6f5f40c6f0 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Wed, 12 Aug 2020 20:41:23 -0400 Subject: DEP: lib: Remove the deprecated financial functions. (#17067) As explained in NEP 32, the financial functions are to be removed from version 1.20. They are now replaced with module level `__getattr__` to give a useful error message for those surprised by the `AttributeError`. This only works for Python 3.7+, but it is expected that by the 1.20 release Python 3.6 will not be supported. --- doc/release/upcoming_changes/17067.expired.rst | 8 + doc/source/reference/routines.financial.rst | 21 - doc/source/reference/routines.rst | 1 - numpy/__init__.py | 21 + numpy/lib/__init__.py | 2 - numpy/lib/financial.py | 967 ------------------------- numpy/lib/tests/test_financial.py | 380 ---------- numpy/lib/tests/test_financial_expired.py | 12 + numpy/tests/test_public_api.py | 1 - tools/functions_missing_types.py | 11 - 10 files changed, 41 insertions(+), 1383 deletions(-) create mode 100644 doc/release/upcoming_changes/17067.expired.rst delete mode 100644 doc/source/reference/routines.financial.rst delete mode 100644 numpy/lib/financial.py delete mode 100644 numpy/lib/tests/test_financial.py create mode 100644 numpy/lib/tests/test_financial_expired.py diff --git a/doc/release/upcoming_changes/17067.expired.rst b/doc/release/upcoming_changes/17067.expired.rst new file mode 100644 index 000000000..a1065d2c3 --- /dev/null +++ b/doc/release/upcoming_changes/17067.expired.rst @@ -0,0 +1,8 @@ +Financial functions removed +--------------------------- +In accordance with NEP 32, the financial functions are removed +from NumPy 1.20. The functions that have been removed are ``fv``, +``ipmt``, ``irr``, ``mirr``, ``nper``, ``npv``, ``pmt``, ``ppmt``, +``pv``, and ``rate``. These functions are available in the +`numpy_financial `_ +library. diff --git a/doc/source/reference/routines.financial.rst b/doc/source/reference/routines.financial.rst deleted file mode 100644 index 5f426d7ab..000000000 --- a/doc/source/reference/routines.financial.rst +++ /dev/null @@ -1,21 +0,0 @@ -Financial functions -******************* - -.. currentmodule:: numpy - -Simple financial functions --------------------------- - -.. autosummary:: - :toctree: generated/ - - fv - pv - npv - pmt - ppmt - ipmt - irr - mirr - nper - rate diff --git a/doc/source/reference/routines.rst b/doc/source/reference/routines.rst index 7a9b97d77..5d6a823b7 100644 --- a/doc/source/reference/routines.rst +++ b/doc/source/reference/routines.rst @@ -28,7 +28,6 @@ indentation. routines.emath routines.err routines.fft - routines.financial routines.functional routines.help routines.indexing diff --git a/numpy/__init__.py b/numpy/__init__.py index 6c4ac98bd..c594928ce 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -214,6 +214,19 @@ else: __all__.remove('Arrayterator') del Arrayterator + # These names were removed in NumPy 1.20. For at least one release, + # attempts to access these names in the numpy namespace will have an + # error message that refers to NEP 32 and points to the numpy_financial + # library. + _financial_names = ['fv', 'ipmt', 'irr', 'mirr', 'nper', 'npv', 'pmt', + 'ppmt', 'pv', 'rate'] + __expired_attrs__ = { + name: (f'In accordance with NEP 32, the function {name} was removed ' + 'from NumPy version 1.20. A replacement for this function ' + 'is available in the numpy_financial library: ' + 'https://pypi.org/project/numpy-financial') + for name in _financial_names} + # Filter out Cython harmless warnings warnings.filterwarnings("ignore", message="numpy.dtype size changed") warnings.filterwarnings("ignore", message="numpy.ufunc size changed") @@ -228,6 +241,14 @@ else: # module level getattr is only supported in 3.7 onwards # https://www.python.org/dev/peps/pep-0562/ def __getattr__(attr): + # Raise AttributeError for expired attributes + try: + msg = __expired_attrs__[attr] + except KeyError: + pass + else: + raise AttributeError(msg) + # Emit warnings for deprecated attributes try: val, msg = __deprecated_attrs__[attr] diff --git a/numpy/lib/__init__.py b/numpy/lib/__init__.py index cb0de0d15..ad88ba347 100644 --- a/numpy/lib/__init__.py +++ b/numpy/lib/__init__.py @@ -35,7 +35,6 @@ from .polynomial import * from .utils import * from .arraysetops import * from .npyio import * -from .financial import * from .arrayterator import Arrayterator from .arraypad import * from ._version import * @@ -54,7 +53,6 @@ __all__ += polynomial.__all__ __all__ += utils.__all__ __all__ += arraysetops.__all__ __all__ += npyio.__all__ -__all__ += financial.__all__ __all__ += nanfunctions.__all__ __all__ += histograms.__all__ diff --git a/numpy/lib/financial.py b/numpy/lib/financial.py deleted file mode 100644 index 709a79dc0..000000000 --- a/numpy/lib/financial.py +++ /dev/null @@ -1,967 +0,0 @@ -"""Some simple financial calculations - -patterned after spreadsheet computations. - -There is some complexity in each function -so that the functions behave like ufuncs with -broadcasting and being able to be called with scalars -or arrays (or other sequences). - -Functions support the :class:`decimal.Decimal` type unless -otherwise stated. -""" -import warnings -from decimal import Decimal -import functools - -import numpy as np -from numpy.core import overrides - - -_depmsg = ("numpy.{name} is deprecated and will be removed from NumPy 1.20. " - "Use numpy_financial.{name} instead " - "(https://pypi.org/project/numpy-financial/).") - -array_function_dispatch = functools.partial( - overrides.array_function_dispatch, module='numpy') - - -__all__ = ['fv', 'pmt', 'nper', 'ipmt', 'ppmt', 'pv', 'rate', - 'irr', 'npv', 'mirr'] - -_when_to_num = {'end':0, 'begin':1, - 'e':0, 'b':1, - 0:0, 1:1, - 'beginning':1, - 'start':1, - 'finish':0} - -def _convert_when(when): - #Test to see if when has already been converted to ndarray - #This will happen if one function calls another, for example ppmt - if isinstance(when, np.ndarray): - return when - try: - return _when_to_num[when] - except (KeyError, TypeError): - return [_when_to_num[x] for x in when] - - -def _fv_dispatcher(rate, nper, pmt, pv, when=None): - warnings.warn(_depmsg.format(name='fv'), - DeprecationWarning, stacklevel=3) - return (rate, nper, pmt, pv) - - -@array_function_dispatch(_fv_dispatcher) -def fv(rate, nper, pmt, pv, when='end'): - """ - Compute the future value. - - .. deprecated:: 1.18 - - `fv` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Given: - * a present value, `pv` - * an interest `rate` compounded once per period, of which - there are - * `nper` total - * a (fixed) payment, `pmt`, paid either - * at the beginning (`when` = {'begin', 1}) or the end - (`when` = {'end', 0}) of each period - - Return: - the value at the end of the `nper` periods - - Parameters - ---------- - rate : scalar or array_like of shape(M, ) - Rate of interest as decimal (not per cent) per period - nper : scalar or array_like of shape(M, ) - Number of compounding periods - pmt : scalar or array_like of shape(M, ) - Payment - pv : scalar or array_like of shape(M, ) - Present value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)). - Defaults to {'end', 0}. - - Returns - ------- - out : ndarray - Future values. If all input is scalar, returns a scalar float. If - any input is array_like, returns future values for each input element. - If multiple inputs are array_like, they all must have the same shape. - - Notes - ----- - The future value is computed by solving the equation:: - - fv + - pv*(1+rate)**nper + - pmt*(1 + rate*when)/rate*((1 + rate)**nper - 1) == 0 - - or, when ``rate == 0``:: - - fv + pv + pmt * nper == 0 - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt - - - Examples - -------- - What is the future value after 10 years of saving $100 now, with - an additional monthly savings of $100. Assume the interest rate is - 5% (annually) compounded monthly? - - >>> np.fv(0.05/12, 10*12, -100, -100) - 15692.928894335748 - - By convention, the negative sign represents cash flow out (i.e. money not - available today). Thus, saving $100 a month at 5% annual interest leads - to $15,692.93 available to spend in 10 years. - - If any input is array_like, returns an array of equal shape. Let's - compare different interest rates from the example above. - - >>> a = np.array((0.05, 0.06, 0.07))/12 - >>> np.fv(a, 10*12, -100, -100) - array([ 15692.92889434, 16569.87435405, 17509.44688102]) # may vary - - """ - when = _convert_when(when) - (rate, nper, pmt, pv, when) = map(np.asarray, [rate, nper, pmt, pv, when]) - temp = (1+rate)**nper - fact = np.where(rate == 0, nper, - (1 + rate*when)*(temp - 1)/rate) - return -(pv*temp + pmt*fact) - - -def _pmt_dispatcher(rate, nper, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='pmt'), - DeprecationWarning, stacklevel=3) - return (rate, nper, pv, fv) - - -@array_function_dispatch(_pmt_dispatcher) -def pmt(rate, nper, pv, fv=0, when='end'): - """ - Compute the payment against loan principal plus interest. - - .. deprecated:: 1.18 - - `pmt` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Given: - * a present value, `pv` (e.g., an amount borrowed) - * a future value, `fv` (e.g., 0) - * an interest `rate` compounded once per period, of which - there are - * `nper` total - * and (optional) specification of whether payment is made - at the beginning (`when` = {'begin', 1}) or the end - (`when` = {'end', 0}) of each period - - Return: - the (fixed) periodic payment. - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - nper : array_like - Number of compounding periods - pv : array_like - Present value - fv : array_like, optional - Future value (default = 0) - when : {{'begin', 1}, {'end', 0}}, {string, int} - When payments are due ('begin' (1) or 'end' (0)) - - Returns - ------- - out : ndarray - Payment against loan plus interest. If all input is scalar, returns a - scalar float. If any input is array_like, returns payment for each - input element. If multiple inputs are array_like, they all must have - the same shape. - - Notes - ----- - The payment is computed by solving the equation:: - - fv + - pv*(1 + rate)**nper + - pmt*(1 + rate*when)/rate*((1 + rate)**nper - 1) == 0 - - or, when ``rate == 0``:: - - fv + pv + pmt * nper == 0 - - for ``pmt``. - - Note that computing a monthly mortgage payment is only - one use for this function. For example, pmt returns the - periodic deposit one must make to achieve a specified - future balance given an initial deposit, a fixed, - periodically compounded interest rate, and the total - number of periods. - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php - ?wg_abbrev=office-formulaOpenDocument-formula-20090508.odt - - Examples - -------- - What is the monthly payment needed to pay off a $200,000 loan in 15 - years at an annual interest rate of 7.5%? - - >>> np.pmt(0.075/12, 12*15, 200000) - -1854.0247200054619 - - In order to pay-off (i.e., have a future-value of 0) the $200,000 obtained - today, a monthly payment of $1,854.02 would be required. Note that this - example illustrates usage of `fv` having a default value of 0. - - """ - when = _convert_when(when) - (rate, nper, pv, fv, when) = map(np.array, [rate, nper, pv, fv, when]) - temp = (1 + rate)**nper - mask = (rate == 0) - masked_rate = np.where(mask, 1, rate) - fact = np.where(mask != 0, nper, - (1 + masked_rate*when)*(temp - 1)/masked_rate) - return -(fv + pv*temp) / fact - - -def _nper_dispatcher(rate, pmt, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='nper'), - DeprecationWarning, stacklevel=3) - return (rate, pmt, pv, fv) - - -@array_function_dispatch(_nper_dispatcher) -def nper(rate, pmt, pv, fv=0, when='end'): - """ - Compute the number of periodic payments. - - .. deprecated:: 1.18 - - `nper` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - :class:`decimal.Decimal` type is not supported. - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - pmt : array_like - Payment - pv : array_like - Present value - fv : array_like, optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)) - - Notes - ----- - The number of periods ``nper`` is computed by solving the equation:: - - fv + pv*(1+rate)**nper + pmt*(1+rate*when)/rate*((1+rate)**nper-1) = 0 - - but if ``rate = 0`` then:: - - fv + pv + pmt*nper = 0 - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - - Examples - -------- - If you only had $150/month to pay towards the loan, how long would it take - to pay-off a loan of $8,000 at 7% annual interest? - - >>> print(np.round(np.nper(0.07/12, -150, 8000), 5)) - 64.07335 - - So, over 64 months would be required to pay off the loan. - - The same analysis could be done with several different interest rates - and/or payments and/or total amounts to produce an entire table. - - >>> np.nper(*(np.ogrid[0.07/12: 0.08/12: 0.01/12, - ... -150 : -99 : 50 , - ... 8000 : 9001 : 1000])) - array([[[ 64.07334877, 74.06368256], - [108.07548412, 127.99022654]], - [[ 66.12443902, 76.87897353], - [114.70165583, 137.90124779]]]) - - """ - when = _convert_when(when) - (rate, pmt, pv, fv, when) = map(np.asarray, [rate, pmt, pv, fv, when]) - - use_zero_rate = False - with np.errstate(divide="raise"): - try: - z = pmt*(1+rate*when)/rate - except FloatingPointError: - use_zero_rate = True - - if use_zero_rate: - return (-fv + pv) / pmt - else: - A = -(fv + pv)/(pmt+0) - B = np.log((-fv+z) / (pv+z))/np.log(1+rate) - return np.where(rate == 0, A, B) - - -def _ipmt_dispatcher(rate, per, nper, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='ipmt'), - DeprecationWarning, stacklevel=3) - return (rate, per, nper, pv, fv) - - -@array_function_dispatch(_ipmt_dispatcher) -def ipmt(rate, per, nper, pv, fv=0, when='end'): - """ - Compute the interest portion of a payment. - - .. deprecated:: 1.18 - - `ipmt` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - rate : scalar or array_like of shape(M, ) - Rate of interest as decimal (not per cent) per period - per : scalar or array_like of shape(M, ) - Interest paid against the loan changes during the life or the loan. - The `per` is the payment period to calculate the interest amount. - nper : scalar or array_like of shape(M, ) - Number of compounding periods - pv : scalar or array_like of shape(M, ) - Present value - fv : scalar or array_like of shape(M, ), optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)). - Defaults to {'end', 0}. - - Returns - ------- - out : ndarray - Interest portion of payment. If all input is scalar, returns a scalar - float. If any input is array_like, returns interest payment for each - input element. If multiple inputs are array_like, they all must have - the same shape. - - See Also - -------- - ppmt, pmt, pv - - Notes - ----- - The total payment is made up of payment against principal plus interest. - - ``pmt = ppmt + ipmt`` - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - - Examples - -------- - What is the amortization schedule for a 1 year loan of $2500 at - 8.24% interest per year compounded monthly? - - >>> principal = 2500.00 - - The 'per' variable represents the periods of the loan. Remember that - financial equations start the period count at 1! - - >>> per = np.arange(1*12) + 1 - >>> ipmt = np.ipmt(0.0824/12, per, 1*12, principal) - >>> ppmt = np.ppmt(0.0824/12, per, 1*12, principal) - - Each element of the sum of the 'ipmt' and 'ppmt' arrays should equal - 'pmt'. - - >>> pmt = np.pmt(0.0824/12, 1*12, principal) - >>> np.allclose(ipmt + ppmt, pmt) - True - - >>> fmt = '{0:2d} {1:8.2f} {2:8.2f} {3:8.2f}' - >>> for payment in per: - ... index = payment - 1 - ... principal = principal + ppmt[index] - ... print(fmt.format(payment, ppmt[index], ipmt[index], principal)) - 1 -200.58 -17.17 2299.42 - 2 -201.96 -15.79 2097.46 - 3 -203.35 -14.40 1894.11 - 4 -204.74 -13.01 1689.37 - 5 -206.15 -11.60 1483.22 - 6 -207.56 -10.18 1275.66 - 7 -208.99 -8.76 1066.67 - 8 -210.42 -7.32 856.25 - 9 -211.87 -5.88 644.38 - 10 -213.32 -4.42 431.05 - 11 -214.79 -2.96 216.26 - 12 -216.26 -1.49 -0.00 - - >>> interestpd = np.sum(ipmt) - >>> np.round(interestpd, 2) - -112.98 - - """ - when = _convert_when(when) - rate, per, nper, pv, fv, when = np.broadcast_arrays(rate, per, nper, - pv, fv, when) - total_pmt = pmt(rate, nper, pv, fv, when) - ipmt = _rbl(rate, per, total_pmt, pv, when)*rate - try: - ipmt = np.where(when == 1, ipmt/(1 + rate), ipmt) - ipmt = np.where(np.logical_and(when == 1, per == 1), 0, ipmt) - except IndexError: - pass - return ipmt - - -def _rbl(rate, per, pmt, pv, when): - """ - This function is here to simply have a different name for the 'fv' - function to not interfere with the 'fv' keyword argument within the 'ipmt' - function. It is the 'remaining balance on loan' which might be useful as - its own function, but is easily calculated with the 'fv' function. - """ - return fv(rate, (per - 1), pmt, pv, when) - - -def _ppmt_dispatcher(rate, per, nper, pv, fv=None, when=None): - warnings.warn(_depmsg.format(name='ppmt'), - DeprecationWarning, stacklevel=3) - return (rate, per, nper, pv, fv) - - -@array_function_dispatch(_ppmt_dispatcher) -def ppmt(rate, per, nper, pv, fv=0, when='end'): - """ - Compute the payment against loan principal. - - .. deprecated:: 1.18 - - `ppmt` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - per : array_like, int - Amount paid against the loan changes. The `per` is the period of - interest. - nper : array_like - Number of compounding periods - pv : array_like - Present value - fv : array_like, optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int} - When payments are due ('begin' (1) or 'end' (0)) - - See Also - -------- - pmt, pv, ipmt - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - - """ - total = pmt(rate, nper, pv, fv, when) - return total - ipmt(rate, per, nper, pv, fv, when) - - -def _pv_dispatcher(rate, nper, pmt, fv=None, when=None): - warnings.warn(_depmsg.format(name='pv'), - DeprecationWarning, stacklevel=3) - return (rate, nper, nper, pv, fv) - - -@array_function_dispatch(_pv_dispatcher) -def pv(rate, nper, pmt, fv=0, when='end'): - """ - Compute the present value. - - .. deprecated:: 1.18 - - `pv` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Given: - * a future value, `fv` - * an interest `rate` compounded once per period, of which - there are - * `nper` total - * a (fixed) payment, `pmt`, paid either - * at the beginning (`when` = {'begin', 1}) or the end - (`when` = {'end', 0}) of each period - - Return: - the value now - - Parameters - ---------- - rate : array_like - Rate of interest (per period) - nper : array_like - Number of compounding periods - pmt : array_like - Payment - fv : array_like, optional - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)) - - Returns - ------- - out : ndarray, float - Present value of a series of payments or investments. - - Notes - ----- - The present value is computed by solving the equation:: - - fv + - pv*(1 + rate)**nper + - pmt*(1 + rate*when)/rate*((1 + rate)**nper - 1) = 0 - - or, when ``rate = 0``:: - - fv + pv + pmt * nper = 0 - - for `pv`, which is then returned. - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt - - Examples - -------- - What is the present value (e.g., the initial investment) - of an investment that needs to total $15692.93 - after 10 years of saving $100 every month? Assume the - interest rate is 5% (annually) compounded monthly. - - >>> np.pv(0.05/12, 10*12, -100, 15692.93) - -100.00067131625819 - - By convention, the negative sign represents cash flow out - (i.e., money not available today). Thus, to end up with - $15,692.93 in 10 years saving $100 a month at 5% annual - interest, one's initial deposit should also be $100. - - If any input is array_like, ``pv`` returns an array of equal shape. - Let's compare different interest rates in the example above: - - >>> a = np.array((0.05, 0.04, 0.03))/12 - >>> np.pv(a, 10*12, -100, 15692.93) - array([ -100.00067132, -649.26771385, -1273.78633713]) # may vary - - So, to end up with the same $15692.93 under the same $100 per month - "savings plan," for annual interest rates of 4% and 3%, one would - need initial investments of $649.27 and $1273.79, respectively. - - """ - when = _convert_when(when) - (rate, nper, pmt, fv, when) = map(np.asarray, [rate, nper, pmt, fv, when]) - temp = (1+rate)**nper - fact = np.where(rate == 0, nper, (1+rate*when)*(temp-1)/rate) - return -(fv + pmt*fact)/temp - -# Computed with Sage -# (y + (r + 1)^n*x + p*((r + 1)^n - 1)*(r*w + 1)/r)/(n*(r + 1)^(n - 1)*x - -# p*((r + 1)^n - 1)*(r*w + 1)/r^2 + n*p*(r + 1)^(n - 1)*(r*w + 1)/r + -# p*((r + 1)^n - 1)*w/r) - -def _g_div_gp(r, n, p, x, y, w): - t1 = (r+1)**n - t2 = (r+1)**(n-1) - return ((y + t1*x + p*(t1 - 1)*(r*w + 1)/r) / - (n*t2*x - p*(t1 - 1)*(r*w + 1)/(r**2) + n*p*t2*(r*w + 1)/r + - p*(t1 - 1)*w/r)) - - -def _rate_dispatcher(nper, pmt, pv, fv, when=None, guess=None, tol=None, - maxiter=None): - warnings.warn(_depmsg.format(name='rate'), - DeprecationWarning, stacklevel=3) - return (nper, pmt, pv, fv) - - -# Use Newton's iteration until the change is less than 1e-6 -# for all values or a maximum of 100 iterations is reached. -# Newton's rule is -# r_{n+1} = r_{n} - g(r_n)/g'(r_n) -# where -# g(r) is the formula -# g'(r) is the derivative with respect to r. -@array_function_dispatch(_rate_dispatcher) -def rate(nper, pmt, pv, fv, when='end', guess=None, tol=None, maxiter=100): - """ - Compute the rate of interest per period. - - .. deprecated:: 1.18 - - `rate` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - nper : array_like - Number of compounding periods - pmt : array_like - Payment - pv : array_like - Present value - fv : array_like - Future value - when : {{'begin', 1}, {'end', 0}}, {string, int}, optional - When payments are due ('begin' (1) or 'end' (0)) - guess : Number, optional - Starting guess for solving the rate of interest, default 0.1 - tol : Number, optional - Required tolerance for the solution, default 1e-6 - maxiter : int, optional - Maximum iterations in finding the solution - - Notes - ----- - The rate of interest is computed by iteratively solving the - (non-linear) equation:: - - fv + pv*(1+rate)**nper + pmt*(1+rate*when)/rate * ((1+rate)**nper - 1) = 0 - - for ``rate``. - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] Wheeler, D. A., E. Rathke, and R. Weir (Eds.) (2009, May). - Open Document Format for Office Applications (OpenDocument)v1.2, - Part 2: Recalculated Formula (OpenFormula) Format - Annotated Version, - Pre-Draft 12. Organization for the Advancement of Structured Information - Standards (OASIS). Billerica, MA, USA. [ODT Document]. - Available: - http://www.oasis-open.org/committees/documents.php?wg_abbrev=office-formula - OpenDocument-formula-20090508.odt - - """ - when = _convert_when(when) - default_type = Decimal if isinstance(pmt, Decimal) else float - - # Handle casting defaults to Decimal if/when pmt is a Decimal and - # guess and/or tol are not given default values - if guess is None: - guess = default_type('0.1') - - if tol is None: - tol = default_type('1e-6') - - (nper, pmt, pv, fv, when) = map(np.asarray, [nper, pmt, pv, fv, when]) - - rn = guess - iterator = 0 - close = False - while (iterator < maxiter) and not close: - rnp1 = rn - _g_div_gp(rn, nper, pmt, pv, fv, when) - diff = abs(rnp1-rn) - close = np.all(diff < tol) - iterator += 1 - rn = rnp1 - if not close: - # Return nan's in array of the same shape as rn - return np.nan + rn - else: - return rn - - -def _irr_dispatcher(values): - warnings.warn(_depmsg.format(name='irr'), - DeprecationWarning, stacklevel=3) - return (values,) - - -@array_function_dispatch(_irr_dispatcher) -def irr(values): - """ - Return the Internal Rate of Return (IRR). - - .. deprecated:: 1.18 - - `irr` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - This is the "average" periodically compounded rate of return - that gives a net present value of 0.0; for a more complete explanation, - see Notes below. - - :class:`decimal.Decimal` type is not supported. - - Parameters - ---------- - values : array_like, shape(N,) - Input cash flows per time period. By convention, net "deposits" - are negative and net "withdrawals" are positive. Thus, for - example, at least the first element of `values`, which represents - the initial investment, will typically be negative. - - Returns - ------- - out : float - Internal Rate of Return for periodic input values. - - Notes - ----- - The IRR is perhaps best understood through an example (illustrated - using np.irr in the Examples section below). Suppose one invests 100 - units and then makes the following withdrawals at regular (fixed) - intervals: 39, 59, 55, 20. Assuming the ending value is 0, one's 100 - unit investment yields 173 units; however, due to the combination of - compounding and the periodic withdrawals, the "average" rate of return - is neither simply 0.73/4 nor (1.73)^0.25-1. Rather, it is the solution - (for :math:`r`) of the equation: - - .. math:: -100 + \\frac{39}{1+r} + \\frac{59}{(1+r)^2} - + \\frac{55}{(1+r)^3} + \\frac{20}{(1+r)^4} = 0 - - In general, for `values` :math:`= [v_0, v_1, ... v_M]`, - irr is the solution of the equation: [2]_ - - .. math:: \\sum_{t=0}^M{\\frac{v_t}{(1+irr)^{t}}} = 0 - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., - Addison-Wesley, 2003, pg. 348. - - Examples - -------- - >>> round(np.irr([-100, 39, 59, 55, 20]), 5) - 0.28095 - >>> round(np.irr([-100, 0, 0, 74]), 5) - -0.0955 - >>> round(np.irr([-100, 100, 0, -7]), 5) - -0.0833 - >>> round(np.irr([-100, 100, 0, 7]), 5) - 0.06206 - >>> round(np.irr([-5, 10.5, 1, -8, 1]), 5) - 0.0886 - - """ - # `np.roots` call is why this function does not support Decimal type. - # - # Ultimately Decimal support needs to be added to np.roots, which has - # greater implications on the entire linear algebra module and how it does - # eigenvalue computations. - res = np.roots(values[::-1]) - mask = (res.imag == 0) & (res.real > 0) - if not mask.any(): - return np.nan - res = res[mask].real - # NPV(rate) = 0 can have more than one solution so we return - # only the solution closest to zero. - rate = 1/res - 1 - rate = rate.item(np.argmin(np.abs(rate))) - return rate - - -def _npv_dispatcher(rate, values): - warnings.warn(_depmsg.format(name='npv'), - DeprecationWarning, stacklevel=3) - return (values,) - - -@array_function_dispatch(_npv_dispatcher) -def npv(rate, values): - """ - Returns the NPV (Net Present Value) of a cash flow series. - - .. deprecated:: 1.18 - - `npv` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - rate : scalar - The discount rate. - values : array_like, shape(M, ) - The values of the time series of cash flows. The (fixed) time - interval between cash flow "events" must be the same as that for - which `rate` is given (i.e., if `rate` is per year, then precisely - a year is understood to elapse between each cash flow event). By - convention, investments or "deposits" are negative, income or - "withdrawals" are positive; `values` must begin with the initial - investment, thus `values[0]` will typically be negative. - - Returns - ------- - out : float - The NPV of the input cash flow series `values` at the discount - `rate`. - - Warnings - -------- - ``npv`` considers a series of cashflows starting in the present (t = 0). - NPV can also be defined with a series of future cashflows, paid at the - end, rather than the start, of each period. If future cashflows are used, - the first cashflow `values[0]` must be zeroed and added to the net - present value of the future cashflows. This is demonstrated in the - examples. - - Notes - ----- - Returns the result of: [2]_ - - .. math :: \\sum_{t=0}^{M-1}{\\frac{values_t}{(1+rate)^{t}}} - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - .. [2] L. J. Gitman, "Principles of Managerial Finance, Brief," 3rd ed., - Addison-Wesley, 2003, pg. 346. - - Examples - -------- - Consider a potential project with an initial investment of $40 000 and - projected cashflows of $5 000, $8 000, $12 000 and $30 000 at the end of - each period discounted at a rate of 8% per period. To find the project's - net present value: - - >>> rate, cashflows = 0.08, [-40_000, 5_000, 8_000, 12_000, 30_000] - >>> np.npv(rate, cashflows).round(5) - 3065.22267 - - It may be preferable to split the projected cashflow into an initial - investment and expected future cashflows. In this case, the value of - the initial cashflow is zero and the initial investment is later added - to the future cashflows net present value: - - >>> initial_cashflow = cashflows[0] - >>> cashflows[0] = 0 - >>> np.round(np.npv(rate, cashflows) + initial_cashflow, 5) - 3065.22267 - - """ - values = np.asarray(values) - return (values / (1+rate)**np.arange(0, len(values))).sum(axis=0) - - -def _mirr_dispatcher(values, finance_rate, reinvest_rate): - warnings.warn(_depmsg.format(name='mirr'), - DeprecationWarning, stacklevel=3) - return (values,) - - -@array_function_dispatch(_mirr_dispatcher) -def mirr(values, finance_rate, reinvest_rate): - """ - Modified internal rate of return. - - .. deprecated:: 1.18 - - `mirr` is deprecated; for details, see NEP 32 [1]_. - Use the corresponding function in the numpy-financial library, - https://pypi.org/project/numpy-financial. - - Parameters - ---------- - values : array_like - Cash flows (must contain at least one positive and one negative - value) or nan is returned. The first value is considered a sunk - cost at time zero. - finance_rate : scalar - Interest rate paid on the cash flows - reinvest_rate : scalar - Interest rate received on the cash flows upon reinvestment - - Returns - ------- - out : float - Modified internal rate of return - - References - ---------- - .. [1] NumPy Enhancement Proposal (NEP) 32, - https://numpy.org/neps/nep-0032-remove-financial-functions.html - """ - values = np.asarray(values) - n = values.size - - # Without this explicit cast the 1/(n - 1) computation below - # becomes a float, which causes TypeError when using Decimal - # values. - if isinstance(finance_rate, Decimal): - n = Decimal(n) - - pos = values > 0 - neg = values < 0 - if not (pos.any() and neg.any()): - return np.nan - numer = np.abs(npv(reinvest_rate, values*pos)) - denom = np.abs(npv(finance_rate, values*neg)) - return (numer/denom)**(1/(n - 1))*(1 + reinvest_rate) - 1 diff --git a/numpy/lib/tests/test_financial.py b/numpy/lib/tests/test_financial.py deleted file mode 100644 index 26e79bc06..000000000 --- a/numpy/lib/tests/test_financial.py +++ /dev/null @@ -1,380 +0,0 @@ -import warnings -from decimal import Decimal - -import numpy as np -from numpy.testing import ( - assert_, assert_almost_equal, assert_allclose, assert_equal, assert_raises - ) - - -def filter_deprecation(func): - def newfunc(*args, **kwargs): - with warnings.catch_warnings(record=True) as ws: - warnings.filterwarnings('always', category=DeprecationWarning) - func(*args, **kwargs) - assert_(all(w.category is DeprecationWarning for w in ws)) - return newfunc - - -class TestFinancial: - @filter_deprecation - def test_npv_irr_congruence(self): - # IRR is defined as the rate required for the present value of a - # a series of cashflows to be zero i.e. NPV(IRR(x), x) = 0 - cashflows = np.array([-40000, 5000, 8000, 12000, 30000]) - assert_allclose(np.npv(np.irr(cashflows), cashflows), 0, atol=1e-10, rtol=0) - - @filter_deprecation - def test_rate(self): - assert_almost_equal( - np.rate(10, 0, -3500, 10000), - 0.1107, 4) - - @filter_deprecation - def test_rate_decimal(self): - rate = np.rate(Decimal('10'), Decimal('0'), Decimal('-3500'), Decimal('10000')) - assert_equal(Decimal('0.1106908537142689284704528100'), rate) - - @filter_deprecation - def test_irr(self): - v = [-150000, 15000, 25000, 35000, 45000, 60000] - assert_almost_equal(np.irr(v), 0.0524, 2) - v = [-100, 0, 0, 74] - assert_almost_equal(np.irr(v), -0.0955, 2) - v = [-100, 39, 59, 55, 20] - assert_almost_equal(np.irr(v), 0.28095, 2) - v = [-100, 100, 0, -7] - assert_almost_equal(np.irr(v), -0.0833, 2) - v = [-100, 100, 0, 7] - assert_almost_equal(np.irr(v), 0.06206, 2) - v = [-5, 10.5, 1, -8, 1] - assert_almost_equal(np.irr(v), 0.0886, 2) - - # Test that if there is no solution then np.irr returns nan - # Fixes gh-6744 - v = [-1, -2, -3] - assert_equal(np.irr(v), np.nan) - - @filter_deprecation - def test_pv(self): - assert_almost_equal(np.pv(0.07, 20, 12000, 0), -127128.17, 2) - - @filter_deprecation - def test_pv_decimal(self): - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0')), - Decimal('-127128.1709461939327295222005')) - - @filter_deprecation - def test_fv(self): - assert_equal(np.fv(0.075, 20, -2000, 0, 0), 86609.362673042924) - - @filter_deprecation - def test_fv_decimal(self): - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), 0, 0), - Decimal('86609.36267304300040536731624')) - - @filter_deprecation - def test_pmt(self): - res = np.pmt(0.08 / 12, 5 * 12, 15000) - tgt = -304.145914 - assert_allclose(res, tgt) - # Test the edge case where rate == 0.0 - res = np.pmt(0.0, 5 * 12, 15000) - tgt = -250.0 - assert_allclose(res, tgt) - # Test the case where we use broadcast and - # the arguments passed in are arrays. - res = np.pmt([[0.0, 0.8], [0.3, 0.8]], [12, 3], [2000, 20000]) - tgt = np.array([[-166.66667, -19311.258], [-626.90814, -19311.258]]) - assert_allclose(res, tgt) - - @filter_deprecation - def test_pmt_decimal(self): - res = np.pmt(Decimal('0.08') / Decimal('12'), 5 * 12, 15000) - tgt = Decimal('-304.1459143262052370338701494') - assert_equal(res, tgt) - # Test the edge case where rate == 0.0 - res = np.pmt(Decimal('0'), Decimal('60'), Decimal('15000')) - tgt = -250 - assert_equal(res, tgt) - # Test the case where we use broadcast and - # the arguments passed in are arrays. - res = np.pmt([[Decimal('0'), Decimal('0.8')], [Decimal('0.3'), Decimal('0.8')]], - [Decimal('12'), Decimal('3')], [Decimal('2000'), Decimal('20000')]) - tgt = np.array([[Decimal('-166.6666666666666666666666667'), Decimal('-19311.25827814569536423841060')], - [Decimal('-626.9081401700757748402586600'), Decimal('-19311.25827814569536423841060')]]) - - # Cannot use the `assert_allclose` because it uses isfinite under the covers - # which does not support the Decimal type - # See issue: https://github.com/numpy/numpy/issues/9954 - assert_equal(res[0][0], tgt[0][0]) - assert_equal(res[0][1], tgt[0][1]) - assert_equal(res[1][0], tgt[1][0]) - assert_equal(res[1][1], tgt[1][1]) - - @filter_deprecation - def test_ppmt(self): - assert_equal(np.round(np.ppmt(0.1 / 12, 1, 60, 55000), 2), -710.25) - - @filter_deprecation - def test_ppmt_decimal(self): - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000')), - Decimal('-710.2541257864217612489830917')) - - # Two tests showing how Decimal is actually getting at a more exact result - # .23 / 12 does not come out nicely as a float but does as a decimal - @filter_deprecation - def test_ppmt_special_rate(self): - assert_equal(np.round(np.ppmt(0.23 / 12, 1, 60, 10000000000), 8), -90238044.232277036) - - @filter_deprecation - def test_ppmt_special_rate_decimal(self): - # When rounded out to 8 decimal places like the float based test, this should not equal the same value - # as the float, substituted for the decimal - def raise_error_because_not_equal(): - assert_equal( - round(np.ppmt(Decimal('0.23') / Decimal('12'), 1, 60, Decimal('10000000000')), 8), - Decimal('-90238044.232277036')) - - assert_raises(AssertionError, raise_error_because_not_equal) - assert_equal(np.ppmt(Decimal('0.23') / Decimal('12'), 1, 60, Decimal('10000000000')), - Decimal('-90238044.2322778884413969909')) - - @filter_deprecation - def test_ipmt(self): - assert_almost_equal(np.round(np.ipmt(0.1 / 12, 1, 24, 2000), 2), -16.67) - - @filter_deprecation - def test_ipmt_decimal(self): - result = np.ipmt(Decimal('0.1') / Decimal('12'), 1, 24, 2000) - assert_equal(result.flat[0], Decimal('-16.66666666666666666666666667')) - - @filter_deprecation - def test_nper(self): - assert_almost_equal(np.nper(0.075, -2000, 0, 100000.), - 21.54, 2) - - @filter_deprecation - def test_nper2(self): - assert_almost_equal(np.nper(0.0, -2000, 0, 100000.), - 50.0, 1) - - @filter_deprecation - def test_npv(self): - assert_almost_equal( - np.npv(0.05, [-15000, 1500, 2500, 3500, 4500, 6000]), - 122.89, 2) - - @filter_deprecation - def test_npv_decimal(self): - assert_equal( - np.npv(Decimal('0.05'), [-15000, 1500, 2500, 3500, 4500, 6000]), - Decimal('122.894854950942692161628715')) - - @filter_deprecation - def test_mirr(self): - val = [-4500, -800, 800, 800, 600, 600, 800, 800, 700, 3000] - assert_almost_equal(np.mirr(val, 0.08, 0.055), 0.0666, 4) - - val = [-120000, 39000, 30000, 21000, 37000, 46000] - assert_almost_equal(np.mirr(val, 0.10, 0.12), 0.126094, 6) - - val = [100, 200, -50, 300, -200] - assert_almost_equal(np.mirr(val, 0.05, 0.06), 0.3428, 4) - - val = [39000, 30000, 21000, 37000, 46000] - assert_(np.isnan(np.mirr(val, 0.10, 0.12))) - - @filter_deprecation - def test_mirr_decimal(self): - val = [Decimal('-4500'), Decimal('-800'), Decimal('800'), Decimal('800'), - Decimal('600'), Decimal('600'), Decimal('800'), Decimal('800'), - Decimal('700'), Decimal('3000')] - assert_equal(np.mirr(val, Decimal('0.08'), Decimal('0.055')), - Decimal('0.066597175031553548874239618')) - - val = [Decimal('-120000'), Decimal('39000'), Decimal('30000'), - Decimal('21000'), Decimal('37000'), Decimal('46000')] - assert_equal(np.mirr(val, Decimal('0.10'), Decimal('0.12')), Decimal('0.126094130365905145828421880')) - - val = [Decimal('100'), Decimal('200'), Decimal('-50'), - Decimal('300'), Decimal('-200')] - assert_equal(np.mirr(val, Decimal('0.05'), Decimal('0.06')), Decimal('0.342823387842176663647819868')) - - val = [Decimal('39000'), Decimal('30000'), Decimal('21000'), Decimal('37000'), Decimal('46000')] - assert_(np.isnan(np.mirr(val, Decimal('0.10'), Decimal('0.12')))) - - @filter_deprecation - def test_when(self): - # begin - assert_equal(np.rate(10, 20, -3500, 10000, 1), - np.rate(10, 20, -3500, 10000, 'begin')) - # end - assert_equal(np.rate(10, 20, -3500, 10000), - np.rate(10, 20, -3500, 10000, 'end')) - assert_equal(np.rate(10, 20, -3500, 10000, 0), - np.rate(10, 20, -3500, 10000, 'end')) - - # begin - assert_equal(np.pv(0.07, 20, 12000, 0, 1), - np.pv(0.07, 20, 12000, 0, 'begin')) - # end - assert_equal(np.pv(0.07, 20, 12000, 0), - np.pv(0.07, 20, 12000, 0, 'end')) - assert_equal(np.pv(0.07, 20, 12000, 0, 0), - np.pv(0.07, 20, 12000, 0, 'end')) - - # begin - assert_equal(np.fv(0.075, 20, -2000, 0, 1), - np.fv(0.075, 20, -2000, 0, 'begin')) - # end - assert_equal(np.fv(0.075, 20, -2000, 0), - np.fv(0.075, 20, -2000, 0, 'end')) - assert_equal(np.fv(0.075, 20, -2000, 0, 0), - np.fv(0.075, 20, -2000, 0, 'end')) - - # begin - assert_equal(np.pmt(0.08 / 12, 5 * 12, 15000., 0, 1), - np.pmt(0.08 / 12, 5 * 12, 15000., 0, 'begin')) - # end - assert_equal(np.pmt(0.08 / 12, 5 * 12, 15000., 0), - np.pmt(0.08 / 12, 5 * 12, 15000., 0, 'end')) - assert_equal(np.pmt(0.08 / 12, 5 * 12, 15000., 0, 0), - np.pmt(0.08 / 12, 5 * 12, 15000., 0, 'end')) - - # begin - assert_equal(np.ppmt(0.1 / 12, 1, 60, 55000, 0, 1), - np.ppmt(0.1 / 12, 1, 60, 55000, 0, 'begin')) - # end - assert_equal(np.ppmt(0.1 / 12, 1, 60, 55000, 0), - np.ppmt(0.1 / 12, 1, 60, 55000, 0, 'end')) - assert_equal(np.ppmt(0.1 / 12, 1, 60, 55000, 0, 0), - np.ppmt(0.1 / 12, 1, 60, 55000, 0, 'end')) - - # begin - assert_equal(np.ipmt(0.1 / 12, 1, 24, 2000, 0, 1), - np.ipmt(0.1 / 12, 1, 24, 2000, 0, 'begin')) - # end - assert_equal(np.ipmt(0.1 / 12, 1, 24, 2000, 0), - np.ipmt(0.1 / 12, 1, 24, 2000, 0, 'end')) - assert_equal(np.ipmt(0.1 / 12, 1, 24, 2000, 0, 0), - np.ipmt(0.1 / 12, 1, 24, 2000, 0, 'end')) - - # begin - assert_equal(np.nper(0.075, -2000, 0, 100000., 1), - np.nper(0.075, -2000, 0, 100000., 'begin')) - # end - assert_equal(np.nper(0.075, -2000, 0, 100000.), - np.nper(0.075, -2000, 0, 100000., 'end')) - assert_equal(np.nper(0.075, -2000, 0, 100000., 0), - np.nper(0.075, -2000, 0, 100000., 'end')) - - @filter_deprecation - def test_decimal_with_when(self): - """Test that decimals are still supported if the when argument is passed""" - # begin - assert_equal(np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), Decimal('1')), - np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), 'begin')) - # end - assert_equal(np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000')), - np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), 'end')) - assert_equal(np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), Decimal('0')), - np.rate(Decimal('10'), Decimal('20'), Decimal('-3500'), Decimal('10000'), 'end')) - - # begin - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), Decimal('1')), - np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), 'begin')) - # end - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0')), - np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), 'end')) - assert_equal(np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), Decimal('0')), - np.pv(Decimal('0.07'), Decimal('20'), Decimal('12000'), Decimal('0'), 'end')) - - # begin - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), Decimal('1')), - np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), 'begin')) - # end - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0')), - np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), 'end')) - assert_equal(np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), Decimal('0')), - np.fv(Decimal('0.075'), Decimal('20'), Decimal('-2000'), Decimal('0'), 'end')) - - # begin - assert_equal(np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), Decimal('1')), - np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), 'begin')) - # end - assert_equal(np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0')), - np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), 'end')) - assert_equal(np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), Decimal('0')), - np.pmt(Decimal('0.08') / Decimal('12'), Decimal('5') * Decimal('12'), Decimal('15000.'), - Decimal('0'), 'end')) - - # begin - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), Decimal('1')), - np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), 'begin')) - # end - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0')), - np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), 'end')) - assert_equal(np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), Decimal('0')), - np.ppmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('60'), Decimal('55000'), - Decimal('0'), 'end')) - - # begin - assert_equal(np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), Decimal('1')).flat[0], - np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), 'begin').flat[0]) - # end - assert_equal(np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0')).flat[0], - np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), 'end').flat[0]) - assert_equal(np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), Decimal('0')).flat[0], - np.ipmt(Decimal('0.1') / Decimal('12'), Decimal('1'), Decimal('24'), Decimal('2000'), - Decimal('0'), 'end').flat[0]) - - @filter_deprecation - def test_broadcast(self): - assert_almost_equal(np.nper(0.075, -2000, 0, 100000., [0, 1]), - [21.5449442, 20.76156441], 4) - - assert_almost_equal(np.ipmt(0.1 / 12, list(range(5)), 24, 2000), - [-17.29165168, -16.66666667, -16.03647345, - -15.40102862, -14.76028842], 4) - - assert_almost_equal(np.ppmt(0.1 / 12, list(range(5)), 24, 2000), - [-74.998201, -75.62318601, -76.25337923, - -76.88882405, -77.52956425], 4) - - assert_almost_equal(np.ppmt(0.1 / 12, list(range(5)), 24, 2000, 0, - [0, 0, 1, 'end', 'begin']), - [-74.998201, -75.62318601, -75.62318601, - -76.88882405, -76.88882405], 4) - - @filter_deprecation - def test_broadcast_decimal(self): - # Use almost equal because precision is tested in the explicit tests, this test is to ensure - # broadcast with Decimal is not broken. - assert_almost_equal(np.ipmt(Decimal('0.1') / Decimal('12'), list(range(5)), Decimal('24'), Decimal('2000')), - [Decimal('-17.29165168'), Decimal('-16.66666667'), Decimal('-16.03647345'), - Decimal('-15.40102862'), Decimal('-14.76028842')], 4) - - assert_almost_equal(np.ppmt(Decimal('0.1') / Decimal('12'), list(range(5)), Decimal('24'), Decimal('2000')), - [Decimal('-74.998201'), Decimal('-75.62318601'), Decimal('-76.25337923'), - Decimal('-76.88882405'), Decimal('-77.52956425')], 4) - - assert_almost_equal(np.ppmt(Decimal('0.1') / Decimal('12'), list(range(5)), Decimal('24'), Decimal('2000'), - Decimal('0'), [Decimal('0'), Decimal('0'), Decimal('1'), 'end', 'begin']), - [Decimal('-74.998201'), Decimal('-75.62318601'), Decimal('-75.62318601'), - Decimal('-76.88882405'), Decimal('-76.88882405')], 4) diff --git a/numpy/lib/tests/test_financial_expired.py b/numpy/lib/tests/test_financial_expired.py new file mode 100644 index 000000000..e1d05da0c --- /dev/null +++ b/numpy/lib/tests/test_financial_expired.py @@ -0,0 +1,12 @@ +import sys +import pytest +import numpy as np + + +def test_financial_expired(): + if sys.version_info[:2] >= (3, 7): + match = 'NEP 32' + else: + match = None + with pytest.raises(AttributeError, match=match): + np.fv diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index a9d6da01c..5b2cbbffe 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -281,7 +281,6 @@ PRIVATE_BUT_PRESENT_MODULES = ['numpy.' + s for s in [ "lib.arraypad", "lib.arraysetops", "lib.arrayterator", - "lib.financial", "lib.function_base", "lib.histograms", "lib.index_tricks", diff --git a/tools/functions_missing_types.py b/tools/functions_missing_types.py index 0fee97777..a32e72dad 100755 --- a/tools/functions_missing_types.py +++ b/tools/functions_missing_types.py @@ -50,17 +50,6 @@ EXCLUDE_LIST = { "object", "str", "unicode", - # Should use numpy_financial instead - "fv", - "ipmt", - "irr", - "mirr", - "nper", - "npv", - "pmt", - "ppmt", - "pv", - "rate", # More standard names should be preferred "alltrue", # all "sometrue", # any -- cgit v1.2.1 From cd1cba812bddc8ae22a664e9347861325f21bb63 Mon Sep 17 00:00:00 2001 From: takanori-pskq Date: Fri, 17 Jul 2020 07:32:13 +0000 Subject: DOC: Fix types including curly braces --- doc/source/reference/c-api/array.rst | 2 +- doc/source/reference/c-api/config.rst | 2 +- doc/source/reference/c-api/dtype.rst | 14 +++++++------- doc/source/reference/c-api/types-and-structures.rst | 4 ++-- doc/source/reference/c-api/ufunc.rst | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index b635c4df2..cfe4d2d51 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -826,7 +826,7 @@ General check of Python Type .. c:function:: PyArray_IsScalar(op, cls) - Evaluates true if *op* is an instance of :c:data:`Py{cls}ArrType_Type`. + Evaluates true if *op* is an instance of ``Py{cls}ArrType_Type``. .. c:function:: PyArray_CheckScalar(op) diff --git a/doc/source/reference/c-api/config.rst b/doc/source/reference/c-api/config.rst index 4592228b5..c3e2c98af 100644 --- a/doc/source/reference/c-api/config.rst +++ b/doc/source/reference/c-api/config.rst @@ -19,7 +19,7 @@ avoid namespace pollution. Data type sizes --------------- -The :c:data:`NPY_SIZEOF_{CTYPE}` constants are defined so that sizeof +The ``NPY_SIZEOF_{CTYPE}`` constants are defined so that sizeof information is available to the pre-processor. .. c:macro:: NPY_SIZEOF_SHORT diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst index 082ecfe97..a04d85212 100644 --- a/doc/source/reference/c-api/dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -30,7 +30,7 @@ Enumerated Types There is a list of enumerated types defined providing the basic 24 data types plus some useful generic names. Whenever the code requires a type number, one of these enumerated types is requested. The types -are all called :c:data:`NPY_{NAME}`: +are all called ``NPY_{NAME}``: .. c:var:: NPY_BOOL @@ -199,7 +199,7 @@ Other useful related constants are The various character codes indicating certain types are also part of an enumerated list. References to type characters (should they be needed at all) should always use these enumerations. The form of them -is :c:data:`NPY_{NAME}LTR` where ``{NAME}`` can be +is ``NPY_{NAME}LTR`` where ``{NAME}`` can be **BOOL**, **BYTE**, **UBYTE**, **SHORT**, **USHORT**, **INT**, **UINT**, **LONG**, **ULONG**, **LONGLONG**, **ULONGLONG**, @@ -247,8 +247,8 @@ Max and min values for integers Number of bits in data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -All :c:data:`NPY_SIZEOF_{CTYPE}` constants have corresponding -:c:data:`NPY_BITSOF_{CTYPE}` constants defined. The :c:data:`NPY_BITSOF_{CTYPE}` +All ``NPY_SIZEOF_{CTYPE}`` constants have corresponding +``NPY_BITSOF_{CTYPE}`` constants defined. The ``NPY_BITSOF_{CTYPE}`` constants provide the number of bits in the data type. Specifically, the available ``{CTYPE}s`` are @@ -263,7 +263,7 @@ All of the numeric data types (integer, floating point, and complex) have constants that are defined to be a specific enumerated type number. Exactly which enumerated type a bit-width type refers to is platform dependent. In particular, the constants available are -:c:data:`PyArray_{NAME}{BITS}` where ``{NAME}`` is **INT**, **UINT**, +``PyArray_{NAME}{BITS}`` where ``{NAME}`` is **INT**, **UINT**, **FLOAT**, **COMPLEX** and ``{BITS}`` can be 8, 16, 32, 64, 80, 96, 128, 160, 192, 256, and 512. Obviously not all bit-widths are available on all platforms for all the kinds of numeric types. Commonly 8-, 16-, @@ -397,8 +397,8 @@ There are also typedefs for signed integers, unsigned integers, floating point, and complex floating point types of specific bit- widths. The available type names are - :c:type:`npy_int{bits}`, :c:type:`npy_uint{bits}`, :c:type:`npy_float{bits}`, - and :c:type:`npy_complex{bits}` + ``npy_int{bits}``, ``npy_uint{bits}``, ``npy_float{bits}``, + and ``npy_complex{bits}`` where ``{bits}`` is the number of bits in the type and can be **8**, **16**, **32**, **64**, 128, and 256 for integer types; 16, **32** diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 5f6fd7d4a..ee57d4680 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -26,7 +26,7 @@ By constructing a new Python type you make available a new object for Python. The ndarray object is an example of a new type defined in C. New types are defined in C by two basic steps: -1. creating a C-structure (usually named :c:type:`Py{Name}Object`) that is +1. creating a C-structure (usually named ``Py{Name}Object``) that is binary- compatible with the :c:type:`PyObject` structure itself but holds the additional information needed for that particular object; @@ -1204,7 +1204,7 @@ ScalarArrayTypes There is a Python type for each of the different built-in data types that can be present in the array Most of these are simple wrappers around the corresponding data type in C. The C-names for these types -are :c:data:`Py{TYPE}ArrType_Type` where ``{TYPE}`` can be +are ``Py{TYPE}ArrType_Type`` where ``{TYPE}`` can be **Bool**, **Byte**, **Short**, **Int**, **Long**, **LongLong**, **UByte**, **UShort**, **UInt**, **ULong**, **ULongLong**, diff --git a/doc/source/reference/c-api/ufunc.rst b/doc/source/reference/c-api/ufunc.rst index abe8935ae..50963c81f 100644 --- a/doc/source/reference/c-api/ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -269,7 +269,7 @@ Functions .. c:function:: int PyUFunc_checkfperr(int errmask, PyObject* errobj) A simple interface to the IEEE error-flag checking support. The - *errmask* argument is a mask of :c:data:`UFUNC_MASK_{ERR}` bitmasks + *errmask* argument is a mask of ``UFUNC_MASK_{ERR}`` bitmasks indicating which errors to check for (and how to check for them). The *errobj* must be a Python tuple with two elements: a string containing the name which will be used in any communication -- cgit v1.2.1 From dd2ddde2a22b80ad8fee815276065f417bdd2177 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Thu, 13 Aug 2020 14:32:39 +0300 Subject: Apply suggestions from code review Co-authored-by: Eric Wieser --- numpy/core/function_base.py | 3 ++- numpy/core/tests/test_function_base.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 46ef6d6ec..b2f17cfeb 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -36,7 +36,8 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, .. versionchanged:: 1.20.0 Values are rounded towards ``-inf`` instead of ``0`` when an - integer ``dtype`` is specified. + integer ``dtype`` is specified. The old behavior can + still be obtained with ``np.linspace(start, stop, num).astype(int)`` Parameters ---------- diff --git a/numpy/core/tests/test_function_base.py b/numpy/core/tests/test_function_base.py index bc51b8144..dad7a5883 100644 --- a/numpy/core/tests/test_function_base.py +++ b/numpy/core/tests/test_function_base.py @@ -407,5 +407,3 @@ class TestLinspace: y = linspace(-1, 3, num=8, dtype=int) t = array([-1, -1, 0, 0, 1, 1, 2, 3], dtype=int) assert_array_equal(y, t) - - -- cgit v1.2.1 From f977f575d6263cac4703d809f57fe895415993ae Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 13 Aug 2020 15:15:56 +0200 Subject: MAINT: Issue the DeprecationWarning after creating the to-be returned array --- numpy/lib/function_base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index e9fe936e9..8e25074ab 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1634,11 +1634,12 @@ def trim_zeros(filt, trim='fb'): "that supports elementwise comparisons with zero" ) warning.__cause__ = ex - warnings.warn(warning, stacklevel=3) - # Fall back to the old implementation if an exception is encountered - # Note that the same exception may or may not be raised here as well - return _trim_zeros_old(filt, trim) + # Fall back to the old implementation if an exception is encountered + # Note that the same exception may or may not be raised here as well + ret = _trim_zeros_old(filt, trim) + warnings.warn(warning, stacklevel=3) + return ret def _trim_zeros_new(filt, trim='fb'): -- cgit v1.2.1 From 0311c93c8c43cd913303c529673d69575901475e Mon Sep 17 00:00:00 2001 From: Sidhant Bansal Date: Thu, 13 Aug 2020 21:32:41 +0800 Subject: ENH: Add support for file like objects to np.core.records.fromfile (#16675) * ENH: Add file like support to np.core.records.fromfile (#2504) Co-authored-by: Ross Barnowski --- doc/release/upcoming_changes/16675.improvement.rst | 4 ++++ numpy/core/records.py | 21 +++++++++++---------- numpy/core/tests/test_records.py | 7 +++++++ 3 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 doc/release/upcoming_changes/16675.improvement.rst diff --git a/doc/release/upcoming_changes/16675.improvement.rst b/doc/release/upcoming_changes/16675.improvement.rst new file mode 100644 index 000000000..bc70d7e0f --- /dev/null +++ b/doc/release/upcoming_changes/16675.improvement.rst @@ -0,0 +1,4 @@ +`numpy.core.records.fromfile` now supports file-like objects +------------------------------------------------------------ +`numpy.rec.fromfile` can now use file-like objects, for instance +:py:class:`io.BytesIO` diff --git a/numpy/core/records.py b/numpy/core/records.py index 0d3fd9118..e95be0e3f 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -40,7 +40,7 @@ from collections import Counter, OrderedDict from . import numeric as sb from . import numerictypes as nt from numpy.compat import ( - isfileobj, os_fspath, contextlib_nullcontext + os_fspath, contextlib_nullcontext ) from numpy.core.overrides import set_module from .arrayprint import get_printoptions @@ -847,13 +847,12 @@ def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None, return _array def get_remaining_size(fd): + pos = fd.tell() try: - fn = fd.fileno() - except AttributeError: - return os.path.getsize(fd.name) - fd.tell() - st = os.fstat(fn) - size = st.st_size - fd.tell() - return size + fd.seek(0, 2) + return fd.tell() - pos + finally: + fd.seek(pos, 0) def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, names=None, titles=None, aligned=False, byteorder=None): @@ -911,7 +910,9 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, elif isinstance(shape, int): shape = (shape,) - if isfileobj(fd): + if hasattr(fd, 'readinto'): + # GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface. + # Example of fd: gzip, BytesIO, BufferedReader # file already opened ctx = contextlib_nullcontext(fd) else: @@ -1036,7 +1037,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None, array('def', dtype=' Date: Thu, 13 Aug 2020 17:06:27 +0200 Subject: TST: Added / updated object array-related tests --- numpy/lib/tests/test_function_base.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 744034e01..ce5c358e0 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1169,9 +1169,10 @@ class TestTrimZeros: a = np.array([0, 0, 1, 0, 2, 3, 4, 0]) b = a.astype(float) c = a.astype(complex) + d = a.astype(object) def values(self): - attr_names = ('a', 'b', 'c') + attr_names = ('a', 'b', 'c', 'd') return (getattr(self, name) for name in attr_names) def test_basic(self): @@ -1207,6 +1208,22 @@ class TestTrimZeros: res = trim_zeros(arr) assert_array_equal(arr, res) + @pytest.mark.parametrize( + 'arr', + [np.array([0, 2**62, 0]), + np.array([0, 2**63, 0]), + np.array([0, 2**64, 0])] + ) + def test_overflow(self, arr): + slc = np.s_[1:2] + res = trim_zeros(arr) + assert_array_equal(res, arr[slc]) + + def test_no_trim(self): + arr = np.array([None, 1, None]) + res = trim_zeros(arr) + assert_array_equal(arr, res) + class TestExtins: -- cgit v1.2.1 From 018ecedfdfd46f9592a4bd24b6d5bc555847d02d Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Thu, 13 Aug 2020 18:35:39 -0300 Subject: DOC: Fixes duplication of toctree content (Closes #17077) --- doc/source/user/tutorials_index.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/source/user/tutorials_index.rst b/doc/source/user/tutorials_index.rst index 5e9419f96..20e2c256c 100644 --- a/doc/source/user/tutorials_index.rst +++ b/doc/source/user/tutorials_index.rst @@ -11,10 +11,6 @@ classes contained in the package, see the :ref:`API reference `. .. toctree:: :maxdepth: 1 - basics - misc - numpy-for-matlab-users tutorial-svd tutorial-ma - building - c-info + -- cgit v1.2.1 From 3166c0d7646efe9d43cb7b363e58db2cf941a5e6 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 12 Aug 2020 19:01:08 +0300 Subject: DOC: first step toward switching themes --- .gitmodules | 3 --- doc/scipy-sphinx-theme | 1 - doc/source/_templates/layout.html | 34 ++++++---------------------------- doc/source/conf.py | 37 +++++++++---------------------------- doc/source/contents.rst | 25 +++++++------------------ doc/source/user/index.rst | 17 +++++++++++++++++ doc_requirements.txt | 1 + 7 files changed, 40 insertions(+), 78 deletions(-) delete mode 160000 doc/scipy-sphinx-theme diff --git a/.gitmodules b/.gitmodules index 1b0706f65..b1e13c3bc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "doc/scipy-sphinx-theme"] - path = doc/scipy-sphinx-theme - url = https://github.com/scipy/scipy-sphinx-theme.git [submodule "doc/sphinxext"] path = doc/sphinxext url = https://github.com/numpy/numpydoc.git diff --git a/doc/scipy-sphinx-theme b/doc/scipy-sphinx-theme deleted file mode 160000 index f0d96ae2b..000000000 --- a/doc/scipy-sphinx-theme +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f0d96ae2bf3b010ce53adadde1e38997497a513e diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 1f8ec518f..a90cad8f6 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -2,38 +2,16 @@ {%- block extrahead %} {{ super() }} {% endblock %} -{%- block header %} -
-
- - NumPy - -
-
- -{% endblock %} {% block rootrellink %} {% if pagename != 'index' %}
  • {{ shorttitle|e }}
    • diff --git a/doc/source/conf.py b/doc/source/conf.py index b908a5a28..e34be7f5c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -94,34 +94,15 @@ def setup(app): # HTML output # ----------------------------------------------------------------------------- -themedir = os.path.join(os.pardir, 'scipy-sphinx-theme', '_theme') -if not os.path.isdir(themedir): - raise RuntimeError("Get the scipy-sphinx-theme first, " - "via git submodule init && git submodule update") - -html_theme = 'scipy' -html_theme_path = [themedir] - -if 'scipyorg' in tags: - # Build for the scipy.org website - html_theme_options = { - "edit_link": True, - "sidebar": "right", - "scipy_org_logo": True, - "rootlinks": [("https://scipy.org/", "Scipy.org"), - ("https://docs.scipy.org/", "Docs")] - } -else: - # Default build - html_theme_options = { - "edit_link": False, - "sidebar": "left", - "scipy_org_logo": False, - "rootlinks": [("https://numpy.org/", "NumPy.org"), - ("https://numpy.org/doc", "Docs"), - ] - } - html_sidebars = {'index': ['indexsidebar.html', 'searchbox.html']} +html_theme = 'pydata_sphinx_theme' + +html_logo = '_static/numpylogo.svg' + +html_theme_options = { + "github_url": "https://github.com/numpy/numpy", + "twitter_url": "https://twitter.com/numpy_team", +} + html_additional_pages = { 'index': 'indexcontent.html', diff --git a/doc/source/contents.rst b/doc/source/contents.rst index baea7784c..953587c6a 100644 --- a/doc/source/contents.rst +++ b/doc/source/contents.rst @@ -5,23 +5,12 @@ NumPy Documentation ################### .. toctree:: + :maxdepth: 1 - user/setting-up - user/quickstart - user/absolute_beginners - user/tutorials_index - user/howtos_index - reference/index - user/explanations_index - f2py/index - glossary - dev/index - dev/underthehood - docs/index - docs/howto_document - benchmarking - bugs - release - about - license + User Guide + API reference + Development +.. This is not really the index page, that is found in + _templates/intexcontent.html The toctree content here will be added to the + top of the template header diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 4e6a29d9f..3a79f0f2e 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -24,3 +24,20 @@ classes contained in the package, see the :ref:`reference`. c-info tutorials_index howtos_index + + +.. These are stuck here to avoid the "WARNING: document isn't included in any + toctree" message + +.. toctree:: + :hidden: + + explanations_index + ../f2py/index + ../glossary + ../dev/underthehood + ../docs/index + ../bugs + ../release + ../about + ../license diff --git a/doc_requirements.txt b/doc_requirements.txt index 6947ec18f..b4da1a4b0 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -3,3 +3,4 @@ ipython scipy matplotlib pandas +pydata-sphinx-theme -- cgit v1.2.1 From 19a8a63ea4194b17a42d21f4d263a13092b26566 Mon Sep 17 00:00:00 2001 From: mattip Date: Wed, 12 Aug 2020 23:54:09 +0300 Subject: rework build to invoke doc_requirements.txt --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a986cba6..d8063e30e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,12 +16,11 @@ jobs: - checkout - run: - name: install dependencies + name: create virtual environment, install dependencies command: | python3 -m venv venv ln -s $(which python3) venv/bin/python3.6 . venv/bin/activate - pip install cython sphinx==2.3.1 matplotlib ipython sudo apt-get update sudo apt-get install -y graphviz texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex @@ -30,10 +29,9 @@ jobs: command: | . venv/bin/activate pip install --upgrade pip 'setuptools<49.2.0' - pip install cython + pip install -r test_requirements.txt pip install . - pip install scipy - pip install pandas + pip install -r doc_requirements.txt - run: name: create release notes -- cgit v1.2.1 From 444ad18dffeaa759c94a4f1f7293f9e05e1f3ffe Mon Sep 17 00:00:00 2001 From: mattip Date: Thu, 13 Aug 2020 07:59:33 +0300 Subject: DOC, NEP: use new theme in NEPs too --- doc/neps/content.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/neps/content.rst diff --git a/doc/neps/content.rst b/doc/neps/content.rst new file mode 100644 index 000000000..a877720f6 --- /dev/null +++ b/doc/neps/content.rst @@ -0,0 +1,21 @@ +===================================== +Roadmap & NumPy Enhancement Proposals +===================================== + +This page provides an overview of development priorities for NumPy. +Specifically, it contains a roadmap with a higher-level overview, as +well as NumPy Enhancement Proposals (NEPs)—suggested changes +to the library—in various stages of discussion or completion (see `NEP +0 `__). + +Roadmap +------- +.. toctree:: + :maxdepth: 1 + + Index + The Scope of NumPy + Current roadmap + Wish list + + -- cgit v1.2.1 From e7acdbe106ad0eff7f6475572ffb1948a5c0b35f Mon Sep 17 00:00:00 2001 From: mattip Date: Thu, 13 Aug 2020 08:02:25 +0300 Subject: BLD: try to store NEP artifacts for preview --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8063e30e..f4ffb5223 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,8 +67,8 @@ jobs: path: doc/build/html/ - # - store_artifacts: - # path: doc/neps/_build/html/ + - store_artifacts: + path: doc/neps/_build/html/ # destination: neps - add_ssh_keys: -- cgit v1.2.1 From 585b39d3739c47ae395e5dda0a21d12a623750c2 Mon Sep 17 00:00:00 2001 From: mattip Date: Thu, 13 Aug 2020 08:15:10 +0300 Subject: DOC, NEP: move wishlist link to the theme configuation --- doc/neps/conf.py | 78 ++++++++++------------------------------------------ doc/neps/content.rst | 6 +++- 2 files changed, 20 insertions(+), 64 deletions(-) diff --git a/doc/neps/conf.py b/doc/neps/conf.py index 6837b12bd..f01ee8a51 100644 --- a/doc/neps/conf.py +++ b/doc/neps/conf.py @@ -45,7 +45,7 @@ templates_path = ['../source/_templates/'] source_suffix = '.rst' # The master toctree document. -master_doc = 'index' +master_doc = 'content' # General information about the project. project = u'NumPy Enhancement Proposals' @@ -82,69 +82,21 @@ todo_include_todos = False ## -- Options for HTML output ---------------------------------------------- # -## The theme to use for HTML and HTML Help pages. See the documentation for -## a list of builtin themes. -## -#html_theme = 'alabaster' -# -## Theme options are theme-specific and customize the look and feel of a theme -## further. For a list of options available for each theme, see the -## documentation. -## -## html_theme_options = {} -# -## Add any paths that contain custom static files (such as style sheets) here, -## relative to this directory. They are copied after the builtin static files, -## so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] -# -## Custom sidebar templates, must be a dictionary that maps document names -## to template names. -## -## This is required for the alabaster theme -## refs: https://alabaster.readthedocs.io/en/latest/installation.html#sidebars -#html_sidebars = { -# '**': [ -# 'relations.html', # needs 'show_related': True theme option to display -# 'searchbox.html', -# ] -#} - -## ----------------------------------------------------------------------------- -# HTML output -# ----------------------------------------------------------------------------- -themedir = os.path.join(os.pardir, 'scipy-sphinx-theme', '_theme') -if not os.path.isdir(themedir): - raise RuntimeError("Get the scipy-sphinx-theme first, " - "via git submodule init && git submodule update") - -html_theme = 'scipy' -html_theme_path = [themedir] - -#if 'scipyorg' in tags: -if True: - # Build for the scipy.org website - html_theme_options = { - "edit_link": True, - "sidebar": "right", - "scipy_org_logo": True, - "rootlinks": [("https://scipy.org/", "Scipy.org"), - ("https://docs.scipy.org/", "Docs")] - } -else: - # Default build - html_theme_options = { - "edit_link": False, - "sidebar": "left", - "scipy_org_logo": False, - "rootlinks": [] - } - html_sidebars = {'index': 'indexsidebar.html'} - -#html_additional_pages = { -# 'index': 'indexcontent.html', -#} +html_theme = 'pydata_sphinx_theme' + +html_logo = '../source/_static/numpylogo.svg' + +html_theme_options = { + "github_url": "https://github.com/numpy/numpy", + "twitter_url": "https://twitter.com/numpy_team", + "external_links": [ + {"name": "Wishlist", + "url": "https://github.com/numpy/numpy/issues?q=is%3Aopen+is%3Aissue+label%3A%2223+-+Wish+List%22", + }, + ], + "show_prev_next": False, +} html_title = "%s" % (project) html_static_path = ['../source/_static'] diff --git a/doc/neps/content.rst b/doc/neps/content.rst index a877720f6..f5d8347c4 100644 --- a/doc/neps/content.rst +++ b/doc/neps/content.rst @@ -16,6 +16,10 @@ Roadmap Index The Scope of NumPy Current roadmap - Wish list + Wishlist (opens new window) |wishlist_link| + +.. |wishlist_link| raw:: html + + WishList -- cgit v1.2.1 From 3905a0f55347655296a37bcb46cc6eee6dd28633 Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 14 Aug 2020 09:26:37 +0300 Subject: DOC: fixes from review --- doc/source/contents.rst | 2 +- doc_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/contents.rst b/doc/source/contents.rst index 953587c6a..5d4e12097 100644 --- a/doc/source/contents.rst +++ b/doc/source/contents.rst @@ -12,5 +12,5 @@ NumPy Documentation Development .. This is not really the index page, that is found in - _templates/intexcontent.html The toctree content here will be added to the + _templates/indexcontent.html The toctree content here will be added to the top of the template header diff --git a/doc_requirements.txt b/doc_requirements.txt index b4da1a4b0..9861ba0a3 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -3,4 +3,4 @@ ipython scipy matplotlib pandas -pydata-sphinx-theme +pydata-sphinx-theme==0.3.1 -- cgit v1.2.1 From 699654a9e763f3bc4d609b5c34fa675076701267 Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 14 Aug 2020 14:28:25 +0300 Subject: MAINT: cleanup layout.html --- doc/source/_templates/layout.html | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index a90cad8f6..1be152265 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -11,22 +11,3 @@ {{ super() }} {% endblock %} - -{% block rootrellink %} - {% if pagename != 'index' %} -
  • {{ shorttitle|e }}
    • - {% endif %} -{% endblock %} - -{% block sidebarsearch %} -{%- if sourcename %} - -{%- endif %} -{{ super() }} -{% endblock %} -- cgit v1.2.1 From 96ccd434925a858312f53aa9757177bf98cb7122 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 09:07:11 +0000 Subject: MAINT: Bump pytest-cov from 2.10.0 to 2.10.1 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.10.0 to 2.10.1. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.10.0...v2.10.1) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 88ee0c64e..c7264f93b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -4,7 +4,7 @@ setuptools<49.2.0 hypothesis==5.23.12 pytest==6.0.1 pytz==2020.1 -pytest-cov==2.10.0 +pytest-cov==2.10.1 pickle5; python_version == '3.7' pickle5; python_version == '3.6' and platform_python_implementation != 'PyPy' # for numpy.random.test.test_extending -- cgit v1.2.1 From 73184a8146e4d8ed2a6ec29d7e0ea995ab96c828 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 09:07:37 +0000 Subject: MAINT: Bump hypothesis from 5.23.12 to 5.26.0 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.23.12 to 5.26.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.23.12...hypothesis-python-5.26.0) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 88ee0c64e..7b868385c 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.23.12 +hypothesis==5.26.0 pytest==6.0.1 pytz==2020.1 pytest-cov==2.10.0 -- cgit v1.2.1 From 8fd3473142364715285e79308c33fff534630c65 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 17 Aug 2020 19:54:41 +0200 Subject: Update fromnumeric.py DOC: rephrase broadcasting rules for clip() --- numpy/core/fromnumeric.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 924690e4b..7f5b3c2a3 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -2066,8 +2066,7 @@ def clip(a, a_min, a_max, out=None, **kwargs): a_max : array_like or None Maximum value. If None, clipping is not performed on upper interval edge. Not more than one of `a_min` and `a_max` may be - None. If `a_min` or `a_max` are array_like, then the three - arrays will be broadcasted to match their shapes. + None. `a_min` and `a_max` are broadcast against `a`. out : ndarray, optional The results will be placed in this array. It may be the input array for in-place clipping. `out` must be of the right shape -- cgit v1.2.1 From 7d7b46c763a86897d254fafcca31e61d08c83ba5 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Fri, 14 Aug 2020 15:20:35 -0700 Subject: NEP: Adjust NEP-35 to make it more user-accessible --- ...array-creation-dispatch-with-array-function.rst | 263 +++++++++++++++------ 1 file changed, 190 insertions(+), 73 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 18a00ae6a..2613ebea2 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -8,16 +8,154 @@ NEP 35 — Array Creation Dispatching With __array_function__ :Status: Draft :Type: Standards Track :Created: 2019-10-15 -:Updated: 2020-08-06 +:Updated: 2020-08-17 :Resolution: Abstract -------- We propose the introduction of a new keyword argument ``like=`` to all array -creation functions to permit dispatching of such functions by the -``__array_function__`` protocol, addressing one of the protocol shortcomings, -as described by NEP-18 [1]_. +creation functions, this argument permits the creation of an array based on +a non-NumPy reference array passed via that argument, resulting in an array +defined by the downstream library implementing that type, which also implements +the ``__array_function__`` protocol. With this we address one of that +protocol's shortcomings, as described by NEP 18 [1]_. + +Motivation and Scope +-------------------- + +Many are the libraries implementing the NumPy API, such as Dask for graph +computing, CuPy for GPGPU computing, xarray for N-D labeled arrays, etc. All +the libraries mentioned have yet another thing in common: they have also adopted +the ``__array_function__`` protocol. The protocol defines a mechanism allowing a +user to directly use the NumPy API as a dispatcher based on the input array +type. In essence, dispatching means users are able to pass a downstream array, +such as a Dask array, directly to one of NumPy's compute functions, and NumPy +will be able to automatically recognize that and send the work back to Dask's +implementation of that function, which will define the return value. For +example: + +.. code:: python + + x = dask.array.arange(5) # Creates dask.array + np.sum(a) # Returns dask.array + +Note above how we called Dask's implementation of ``sum`` via the NumPy +namespace by calling ``np.sum``, and the same would apply if we had a CuPy +array or any other array from a library that adopts ``__array_function__``. +This allows writing code that is agnostic to the implementation library, thus +users can write their code once and still be able to use different array +implementations according to their needs. + +Unfortunately, ``__array_function__`` has limitations, one of them being array +creation functions. In the example above, NumPy was able to call Dask's +implementation because the input array was a Dask array. The same is not true +for array creation functions, in the example the input of ``arange`` is simply +the integer ``5``, not providing any information of the array type that should +be the result, that's where a reference array passed by the ``like=`` argument +proposed here can be of help, as it provides NumPy with the information +required to create the expected type of array. + +The new ``like=`` keyword proposed is solely intended to identify the downstream +library where to dispatch and the object is used only as reference, meaning that +no modifications, copies or processing will be performed on that object. + +We expect that this functionality will be mostly useful to library developers, +allowing them to create new arrays for internal usage based on arrays passed +by the user, preventing unnecessary creation of NumPy arrays that will +ultimately lead to an additional conversion into a downstream array type. + +Support for Python 2.7 has been dropped since NumPy 1.17, therefore we make use +of the keyword-only argument standard described in PEP-3102 [2]_ to implement +``like=``, thus preventing it from being passed by position. + +.. _neps.like-kwarg.usage-and-impact: + +Usage and Impact +---------------- + +To understand the intended use for ``like=``, and before we move to more complex +cases, consider the following illustrative example consisting only of NumPy and +CuPy arrays: + +.. code:: python + + import numpy as np + import cupy + + def my_pad(arr, padding): + padding = np.array(padding, like=arr) + return np.concatenate((padding, arr, padding)) + + my_pad(np.arange(5), [-1, -1]) # Returns np.ndarray + my_pad(cupy.arange(5), [-1, -1]) # Returns cupy.core.core.ndarray + +Note in the ``my_pad`` function above how ``arr`` is used as a reference to +dictate what array type padding should have, before concatenating the arrays to +produce the result. On the other hand, if ``like=`` wasn't used, the NumPy case +case would still work, but CuPy wouldn't allow this kind of automatic +conversion, ultimately raising a +``TypeError: Only cupy arrays can be concatenated`` exception. + +Now we should look at how a library like Dask could benefit from ``like=``. +Before we understand that, it's important to understand a bit about Dask basics +and ensures correctness with ``__array_function__``. Note that Dask can compute +different sorts of objects, like dataframes, bags and arrays, here we will focus +strictly on arrays, which are the objects we can use ``__array_function__`` +with. + +Dask uses a graph computing model, meaning it breaks down a large problem in +many smaller problems and merge their results to reach the final result. To +break the problem down into smaller ones, Dask also breaks arrays into smaller +arrays, that it calls "chunks". A Dask array can thus consist of one or more +chunks and they may be of different types. However, in the context of +``__array_function__``, Dask only allows chunks of the same type, for example, +a Dask array can be formed of several NumPy arrays or several CuPy arrays, but +not a mix of both. + +To avoid mismatched types during compute, Dask keeps an attribute ``_meta`` as +part of its array throughout computation, this attribute is used to both predict +the output type at graph creation time and to create any intermediary arrays +that are necessary within some function's computation. Going back to our +previous example, we can use ``_meta`` information to identify what kind of +array we would use for padding, as seen below: + +.. code:: python + + import numpy as np + import cupy + import dask.array as da + from dask.array.utils import meta_from_array + + def my_pad(arr, padding): + padding = np.array(padding, like=meta_from_array(arr)) + return np.concatenate((padding, arr, padding)) + + # Returns dask.array + my_pad(da.arange(5), [-1, -1]) + + # Returns dask.array + my_pad(da.from_array(cupy.arange(5)), [-1, -1]) + +Note how ``chunktype`` in the return value above changes from +``numpy.ndarray`` in the first ``my_pad`` call to ``cupy.ndarray`` in the +second. + +To enable proper identification of the array type we use Dask's utility function +``meta_from_array``, which was introduced as part of the work to support +``__array_function__``, allowing Dask to handle ``_meta`` appropriately. That +function is primarily targeted at the library's internal usage to ensure chunks +are created with correct types. Without the ``like=`` argument, it would be +impossible to ensure ``my_pad`` creates a padding array with a type matching +that of the input array, which would cause cause a ``TypeError`` exception to +be raised by CuPy, as discussed above would happen to the CuPy case alone. + +Backward Compatibility +---------------------- + +This proposal does not raise any backward compatibility issues within NumPy, +given that it only introduces a new keyword argument to existing array creation +functions with a default ``None`` value, thus not changing current behavior. Detailed description -------------------- @@ -28,10 +166,6 @@ did not -- and did not intend to -- address the creation of arrays by downstream libraries, preventing those libraries from using such important functionality in that context. -Other NEPs have been written to address parts of that limitation, such as the -introduction of the ``__duckarray__`` protocol in NEP-30 [2]_, and the -introduction of an overriding mechanism called ``uarray`` by NEP-31 [3]_. - The purpose of this NEP is to address that shortcoming in a simple and straighforward way: introduce a new ``like=`` keyword argument, similar to how the ``empty_like`` family of functions work. When array creation functions @@ -39,25 +173,25 @@ receive such an argument, they will trigger the ``__array_function__`` protocol, and call the downstream library's own array creation function implementation. The ``like=`` argument, as its own name suggests, shall be used solely for the purpose of identifying where to dispatch. In contrast to the way -``__array_function__`` has been used so far (the first argument identifies where -to dispatch), and to avoid breaking NumPy's API with regards to array creation, -the new ``like=`` keyword shall be used for the purpose of dispatching. - -Usage Guidance -~~~~~~~~~~~~~~ - -The new ``like=`` keyword is solely intended to identify the downstream library -where to dispatch and the object is used only as reference, meaning that no -modifications, copies or processing will be performed on that object. - -We expect that this functionality will be mostly useful to library developers, -allowing them to create new arrays for internal usage based on arrays passed -by the user, preventing unnecessary creation of NumPy arrays that will -ultimately lead to an additional conversion into a downstream array type. - -Support for Python 2.7 has been dropped since NumPy 1.17, therefore we should -make use of the keyword-only argument standard described in PEP-3102 [4]_ to -implement the ``like=``, thus preventing it from being passed by position. +``__array_function__`` has been used so far (the first argument identifies the +target downstream library), and to avoid breaking NumPy's API with regards to +array creation, the new ``like=`` keyword shall be used for the purpose of +dispatching. + +Downstream libraries will benefit from the ``like=`` argument without any +changes to their API, given the argument is of exclusive implementation in +NumPy. It will still be required that downstream libraries implement the +``__array_function__`` protocol, as described by NEP 18 [1]_, and appropriately +introduce the argument to their calls to NumPy array creation functions, as +exemplified in :ref:`neps.like-kwarg.usage-and-impact`. + +Related work +------------ + +Other NEPs have been written to address parts of ``__array_function__`` +protocol's limitation, such as the introduction of the ``__duckarray__`` +protocol in NEP 30 [3]_, and the introduction of an overriding mechanism called +``uarray`` by NEP 31 [4]_. Implementation -------------- @@ -66,10 +200,10 @@ The implementation requires introducing a new ``like=`` keyword to all existing array creation functions of NumPy. As examples of functions that would add this new argument (but not limited to) we can cite those taking array-like objects such as ``array`` and ``asarray``, functions that create arrays based on -numerical ranges such as ``range`` and ``linspace``, as well as the ``empty`` -family of functions, even though that may be redundant, since there exists -already specializations for those with the naming format ``empty_like``. As of -the writing of this NEP, a complete list of array creation functions can be +numerical inputs such as ``range`` and ``identity``, as well as the ``empty`` +family of functions, even though that may be redundant, since specializations +for those already exist with the naming format ``empty_like``. As of the +writing of this NEP, a complete list of array creation functions can be found in [5]_. This newly proposed keyword shall be removed by the ``__array_function__`` @@ -135,60 +269,43 @@ There are two downsides to the implementation above for C functions: 2. To follow current implementation standards, documentation should be attached directly to the Python source code. -Alternatively for C functions, the implementation of ``like=`` could be moved -into the C implementation itself. This is not the primary suggestion here due -to its inherent complexity which would be difficult too long to describe in its -entirety here, and too tedious for the reader. However, we leave that as an -option open for discussion. +The first version of this proposal suggested the C implementation above as one +viable solution. However, due to the downsides pointed above we have decided to +implement that entirely in C. Please refer to [implementation]_ for details. -Usage ------ +Alternatives +------------ -The purpose of this NEP is to keep things simple. Similarly, we can exemplify -the usage of ``like=`` in a simple way. Imagine you have an array of ones -created by a downstream library, such as CuPy. What you need now is a new array -that can be created using the NumPy API, but that will in fact be created by -the downstream library, a simple way to achieve that is shown below. +Recently a new protocol to replace ``__array_function__`` entirely was proposed +by NEP 37 [6]_, which would require considerable rework by downstream libraries +that adopt ``__array_function__`` already, because of that we still believe the +``like=`` argument is beneficial for NumPy and downstream libraries. However, +that proposal wouldn't necessarily be considered a direct alternative to the +present NEP, as it would replace NEP 18 entirely, on which this builds upon. +Discussion on details about this new proposal and why that would require rework +by downstream libraries is beyond the scopy of the present proposal. -.. code:: python - - x = cupy.ones(2) - np.array([1, 3, 5], like=x) # Returns cupy.ndarray - -As a second example, we could also create an array of evenly spaced numbers -using a Dask identity matrix as reference: - -.. code:: python - - x = dask.array.eye(3) - np.linspace(0, 2, like=x) # Returns dask.array +Discussion +---------- +.. [implementation] `Implementation's pull request on GitHub `_ +.. [discussion] `Further discussion on implementation and the NEP's content `_ -Compatibility -------------- +References +---------- -This proposal does not raise any backward compatibility issues within NumPy, -given that it only introduces a new keyword argument to existing array creation -functions. - -Downstream libraries will benefit from the ``like=`` argument automatically, -that is, without any explicit changes in their codebase. The only requirement -is that they already implement the ``__array_function__`` protocol, as -described by NEP-18 [2]_. +.. [1] `NEP 18 - A dispatch mechanism for NumPy's high level array functions `_. -References and Footnotes ------------------------- +.. [2] `PEP 3102 — Keyword-Only Arguments `_. -.. [1] `NEP-18 - A dispatch mechanism for NumPy's high level array functions `_. +.. [3] `NEP 30 — Duck Typing for NumPy Arrays - Implementation `_. -.. [2] `NEP 30 — Duck Typing for NumPy Arrays - Implementation `_. - -.. [3] `NEP 31 — Context-local and global overrides of the NumPy API `_. - -.. [4] `PEP 3102 — Keyword-Only Arguments `_. +.. [4] `NEP 31 — Context-local and global overrides of the NumPy API `_. .. [5] `Array creation routines `_. +.. [6] `NEP 37 — A dispatch protocol for NumPy-like modules `_. + Copyright --------- -- cgit v1.2.1 From 9b660e445bb19331b8d4308223b9da418166ef80 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Mon, 17 Aug 2020 14:37:07 -0700 Subject: NEP: Simplify NEP-35 further with reviewer's suggestions --- ...array-creation-dispatch-with-array-function.rst | 56 +++++++++++++++------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 2613ebea2..8554cbccd 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -27,34 +27,49 @@ Motivation and Scope Many are the libraries implementing the NumPy API, such as Dask for graph computing, CuPy for GPGPU computing, xarray for N-D labeled arrays, etc. All the libraries mentioned have yet another thing in common: they have also adopted -the ``__array_function__`` protocol. The protocol defines a mechanism allowing a -user to directly use the NumPy API as a dispatcher based on the input array -type. In essence, dispatching means users are able to pass a downstream array, -such as a Dask array, directly to one of NumPy's compute functions, and NumPy -will be able to automatically recognize that and send the work back to Dask's -implementation of that function, which will define the return value. For -example: +the ``__array_function__`` protocol; a protocol that allows NumPy to understand +and treat downstream objects as if they are the native ``numpy.ndarray`` object. +Hence the community while using various libraries still benefits from a unified +NumPy API. This not only brings great convenience for standardization but also +removes the burden of learning a new API and rewriting code for every new +object. In more technical terms, this mechanism of the protocol is called a +"dispatcher", which is the terminology we use from here onwards when referring +to that. + .. code:: python x = dask.array.arange(5) # Creates dask.array - np.sum(a) # Returns dask.array + np.diff(x) # Returns dask.array Note above how we called Dask's implementation of ``sum`` via the NumPy -namespace by calling ``np.sum``, and the same would apply if we had a CuPy +namespace by calling ``np.diff``, and the same would apply if we had a CuPy array or any other array from a library that adopts ``__array_function__``. This allows writing code that is agnostic to the implementation library, thus users can write their code once and still be able to use different array implementations according to their needs. -Unfortunately, ``__array_function__`` has limitations, one of them being array -creation functions. In the example above, NumPy was able to call Dask's -implementation because the input array was a Dask array. The same is not true -for array creation functions, in the example the input of ``arange`` is simply -the integer ``5``, not providing any information of the array type that should -be the result, that's where a reference array passed by the ``like=`` argument -proposed here can be of help, as it provides NumPy with the information -required to create the expected type of array. +Obviously, having a protocol in-place is useful if the arrays are created +elsewhere and let NumPy handle them. But still these arrays have to be started +in their native library and brought back. Instead if it was possible to create +these objects through NumPy API then there would be an almost complete +experience, all using NumPy syntax. For example, say we have some CuPy array +``cp_arr`` , and want a similar CuPy array with identity matrix. We could still +write the following: + +.. code:: python + x = cupy.identity(3) + +Instead, the better way would be using to only use the NumPy API, this could now +be achieved with: + +.. code:: python + x = np.identity(3, like=cp_arr) + +As if by magic, ``x`` will also be a CuPy array, as NumPy was capable to infer +that from the type of ``cp_arr``. Note that this last step would not be possible +without ``like=``, as it would be impossible for the NumPy to know the user +expects a CuPy array based only on the integer input. The new ``like=`` keyword proposed is solely intended to identify the downstream library where to dispatch and the object is used only as reference, meaning that @@ -150,6 +165,13 @@ impossible to ensure ``my_pad`` creates a padding array with a type matching that of the input array, which would cause cause a ``TypeError`` exception to be raised by CuPy, as discussed above would happen to the CuPy case alone. +Current NumPy users who don't use other arrays from downstream libraries should +have no impact in their current usage of the NumPy API. In the event of the +user passing a NumPy array to ``like=``, that will continue to work as if no +array was passed via that argument. However, this is advised against, as +internally there will be additional checks required that will have an impact in +performance. + Backward Compatibility ---------------------- -- cgit v1.2.1 From 0dc55882a976630d832ebebdb58350d9c26205fe Mon Sep 17 00:00:00 2001 From: jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> Date: Tue, 18 Aug 2020 06:29:02 +0200 Subject: BUG: fix typo in polydiv that prevented promotion to poly1d (#17053) Fix bug caused by typo and added tests --- numpy/lib/polynomial.py | 2 +- numpy/lib/tests/test_polynomial.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 1c124cc0e..7b89eeb70 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -1017,7 +1017,7 @@ def polydiv(u, v): (array([1.5 , 1.75]), array([0.25])) """ - truepoly = (isinstance(u, poly1d) or isinstance(u, poly1d)) + truepoly = (isinstance(u, poly1d) or isinstance(v, poly1d)) u = atleast_1d(u) + 0.0 v = atleast_1d(v) + 0.0 # w has the common type diff --git a/numpy/lib/tests/test_polynomial.py b/numpy/lib/tests/test_polynomial.py index cd0b90dc4..ab6691b43 100644 --- a/numpy/lib/tests/test_polynomial.py +++ b/numpy/lib/tests/test_polynomial.py @@ -243,6 +243,15 @@ class TestPolynomial: assert_equal(q.coeffs.dtype, np.complex128) assert_equal(r.coeffs.dtype, np.complex128) assert_equal(q*a + r, b) + + c = [1, 2, 3] + d = np.poly1d([1, 2, 3]) + s, t = np.polydiv(c, d) + assert isinstance(s, np.poly1d) + assert isinstance(t, np.poly1d) + u, v = np.polydiv(d, c) + assert isinstance(u, np.poly1d) + assert isinstance(v, np.poly1d) def test_poly_coeffs_mutable(self): """ Coefficients should be modifiable """ -- cgit v1.2.1 From 40c7e51f61b014f84debf6ad870a8078b6df1be0 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 18 Aug 2020 11:24:27 +0200 Subject: DOC: simplify documentation of clip() --- numpy/core/fromnumeric.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index 7f5b3c2a3..b07def736 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -2059,14 +2059,10 @@ def clip(a, a_min, a_max, out=None, **kwargs): ---------- a : array_like Array containing elements to clip. - a_min : array_like or None - Minimum value. If None, clipping is not performed on lower - interval edge. Not more than one of `a_min` and `a_max` may be - None. - a_max : array_like or None - Maximum value. If None, clipping is not performed on upper - interval edge. Not more than one of `a_min` and `a_max` may be - None. `a_min` and `a_max` are broadcast against `a`. + a_min, a_max : array_like or None + Minimum and maximum value. If ``None``, clipping is not performed on + the corresponding edge. Only one of `a_min` and `a_max` may be + ``None``. Both are broadcast against `a`. out : ndarray, optional The results will be placed in this array. It may be the input array for in-place clipping. `out` must be of the right shape -- cgit v1.2.1 From 1ef217b19161e01672e243d21263e517f63d817a Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 19 Aug 2020 14:49:16 +0200 Subject: ENH: Add placeholder stubs for all sub-modules --- numpy/__init__.pyi | 37 +++++++++++++++++++++++++++++------- numpy/char.pyi | 4 ++++ numpy/compat/__init__.pyi | 4 ++++ numpy/core/__init__.pyi | 4 ++++ numpy/ctypeslib.pyi | 4 ++++ numpy/emath.pyi | 4 ++++ numpy/fft/__init__.pyi | 4 ++++ numpy/lib/__init__.pyi | 4 ++++ numpy/linalg/__init__.pyi | 4 ++++ numpy/ma/__init__.pyi | 4 ++++ numpy/math.pyi | 4 ++++ numpy/matrixlib/__init__.pyi | 4 ++++ numpy/os.pyi | 4 ++++ numpy/polynomial/__init__.pyi | 4 ++++ numpy/random/__init__.pyi | 4 ++++ numpy/rec.pyi | 4 ++++ numpy/sys.pyi | 4 ++++ numpy/testing/__init__.pyi | 4 ++++ numpy/tests/typing/reveal/modules.py | 26 +++++++++++++++++++++++++ numpy/version.pyi | 4 ++++ numpy/warnings.pyi | 4 ++++ 21 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 numpy/char.pyi create mode 100644 numpy/compat/__init__.pyi create mode 100644 numpy/core/__init__.pyi create mode 100644 numpy/ctypeslib.pyi create mode 100644 numpy/emath.pyi create mode 100644 numpy/fft/__init__.pyi create mode 100644 numpy/lib/__init__.pyi create mode 100644 numpy/linalg/__init__.pyi create mode 100644 numpy/ma/__init__.pyi create mode 100644 numpy/math.pyi create mode 100644 numpy/matrixlib/__init__.pyi create mode 100644 numpy/os.pyi create mode 100644 numpy/polynomial/__init__.pyi create mode 100644 numpy/random/__init__.pyi create mode 100644 numpy/rec.pyi create mode 100644 numpy/sys.pyi create mode 100644 numpy/testing/__init__.pyi create mode 100644 numpy/tests/typing/reveal/modules.py create mode 100644 numpy/version.pyi create mode 100644 numpy/warnings.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index fad5e1774..a1549e2f8 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1,5 +1,5 @@ import builtins -import sys +import sys as _sys import datetime as dt from abc import abstractmethod @@ -33,17 +33,40 @@ from typing import ( Union, ) -if sys.version_info[0] < 3: +if _sys.version_info[0] < 3: class SupportsBytes: ... else: from typing import SupportsBytes -if sys.version_info >= (3, 8): +if _sys.version_info >= (3, 8): from typing import Literal, Protocol else: from typing_extensions import Literal, Protocol +# Ensures that the stubs are picked up +from . import ( + char, + compat, + core, + ctypeslib, + emath, + fft, + lib, + linalg, + ma, + math, + matrixlib, + os, + polynomial, + random, + rec, + sys, + testing, + version, + warnings, +) + # TODO: remove when the full numpy namespace is defined def __getattr__(name: str) -> Any: ... @@ -187,7 +210,7 @@ class _ArrayOrScalarCommon( def __int__(self) -> int: ... def __float__(self) -> float: ... def __complex__(self) -> complex: ... - if sys.version_info[0] < 3: + if _sys.version_info[0] < 3: def __oct__(self) -> str: ... def __hex__(self) -> str: ... def __nonzero__(self) -> bool: ... @@ -214,7 +237,7 @@ class _ArrayOrScalarCommon( def __mul__(self, other): ... def __rmul__(self, other): ... def __imul__(self, other): ... - if sys.version_info[0] < 3: + if _sys.version_info[0] < 3: def __div__(self, other): ... def __rdiv__(self, other): ... def __idiv__(self, other): ... @@ -248,7 +271,7 @@ class _ArrayOrScalarCommon( def __or__(self, other): ... def __ror__(self, other): ... def __ior__(self, other): ... - if sys.version_info[:2] >= (3, 5): + if _sys.version_info[:2] >= (3, 5): def __matmul__(self, other): ... def __rmatmul__(self, other): ... def __neg__(self: _ArraySelf) -> _ArraySelf: ... @@ -426,7 +449,7 @@ class timedelta64(signedinteger): @overload def __add__(self, other: datetime64) -> datetime64: ... def __sub__(self, other: Union[timedelta64, int]) -> timedelta64: ... - if sys.version_info[0] < 3: + if _sys.version_info[0] < 3: @overload def __div__(self, other: timedelta64) -> float: ... @overload diff --git a/numpy/char.pyi b/numpy/char.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/char.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/compat/__init__.pyi b/numpy/compat/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/compat/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/core/__init__.pyi b/numpy/core/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/core/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/ctypeslib.pyi b/numpy/ctypeslib.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/ctypeslib.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/emath.pyi b/numpy/emath.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/emath.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/fft/__init__.pyi b/numpy/fft/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/fft/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/lib/__init__.pyi b/numpy/lib/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/lib/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/linalg/__init__.pyi b/numpy/linalg/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/linalg/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/ma/__init__.pyi b/numpy/ma/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/ma/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/math.pyi b/numpy/math.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/math.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/matrixlib/__init__.pyi b/numpy/matrixlib/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/matrixlib/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/os.pyi b/numpy/os.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/os.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/polynomial/__init__.pyi b/numpy/polynomial/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/polynomial/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/random/__init__.pyi b/numpy/random/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/random/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/rec.pyi b/numpy/rec.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/rec.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/sys.pyi b/numpy/sys.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/sys.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/testing/__init__.pyi b/numpy/testing/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/testing/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/tests/typing/reveal/modules.py b/numpy/tests/typing/reveal/modules.py new file mode 100644 index 000000000..b4a94e326 --- /dev/null +++ b/numpy/tests/typing/reveal/modules.py @@ -0,0 +1,26 @@ +import numpy as np + +reveal_type(np) # E: ModuleType + +reveal_type(np.char) # E: ModuleType +reveal_type(np.compat) # E: ModuleType +reveal_type(np.core) # E: ModuleType +reveal_type(np.ctypeslib) # E: ModuleType +reveal_type(np.emath) # E: ModuleType +reveal_type(np.fft) # E: ModuleType +reveal_type(np.lib) # E: ModuleType +reveal_type(np.linalg) # E: ModuleType +reveal_type(np.ma) # E: ModuleType +reveal_type(np.math) # E: ModuleType +reveal_type(np.matrixlib) # E: ModuleType +reveal_type(np.os) # E: ModuleType +reveal_type(np.polynomial) # E: ModuleType +reveal_type(np.random) # E: ModuleType +reveal_type(np.rec) # E: ModuleType +reveal_type(np.sys) # E: ModuleType +reveal_type(np.testing) # E: ModuleType +reveal_type(np.version) # E: ModuleType +reveal_type(np.warnings) # E: ModuleType + +# TODO: Remove when annotations have been added to `np.linalg.norm` +reveal_type(np.linalg.norm) # E: Any diff --git a/numpy/version.pyi b/numpy/version.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/version.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/warnings.pyi b/numpy/warnings.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/warnings.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... -- cgit v1.2.1 From caf9722d16a2033e56ba77b7f786032b92d49055 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 19 Aug 2020 15:14:30 +0200 Subject: REV: Removed the (accidently included) builtin `sys` module --- numpy/__init__.pyi | 15 +++++++-------- numpy/sys.pyi | 4 ---- numpy/tests/typing/reveal/modules.py | 1 - 3 files changed, 7 insertions(+), 13 deletions(-) delete mode 100644 numpy/sys.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index a1549e2f8..005a15613 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1,5 +1,5 @@ import builtins -import sys as _sys +import sys import datetime as dt from abc import abstractmethod @@ -33,13 +33,13 @@ from typing import ( Union, ) -if _sys.version_info[0] < 3: +if sys.version_info[0] < 3: class SupportsBytes: ... else: from typing import SupportsBytes -if _sys.version_info >= (3, 8): +if sys.version_info >= (3, 8): from typing import Literal, Protocol else: from typing_extensions import Literal, Protocol @@ -61,7 +61,6 @@ from . import ( polynomial, random, rec, - sys, testing, version, warnings, @@ -210,7 +209,7 @@ class _ArrayOrScalarCommon( def __int__(self) -> int: ... def __float__(self) -> float: ... def __complex__(self) -> complex: ... - if _sys.version_info[0] < 3: + if sys.version_info[0] < 3: def __oct__(self) -> str: ... def __hex__(self) -> str: ... def __nonzero__(self) -> bool: ... @@ -237,7 +236,7 @@ class _ArrayOrScalarCommon( def __mul__(self, other): ... def __rmul__(self, other): ... def __imul__(self, other): ... - if _sys.version_info[0] < 3: + if sys.version_info[0] < 3: def __div__(self, other): ... def __rdiv__(self, other): ... def __idiv__(self, other): ... @@ -271,7 +270,7 @@ class _ArrayOrScalarCommon( def __or__(self, other): ... def __ror__(self, other): ... def __ior__(self, other): ... - if _sys.version_info[:2] >= (3, 5): + if sys.version_info[:2] >= (3, 5): def __matmul__(self, other): ... def __rmatmul__(self, other): ... def __neg__(self: _ArraySelf) -> _ArraySelf: ... @@ -449,7 +448,7 @@ class timedelta64(signedinteger): @overload def __add__(self, other: datetime64) -> datetime64: ... def __sub__(self, other: Union[timedelta64, int]) -> timedelta64: ... - if _sys.version_info[0] < 3: + if sys.version_info[0] < 3: @overload def __div__(self, other: timedelta64) -> float: ... @overload diff --git a/numpy/sys.pyi b/numpy/sys.pyi deleted file mode 100644 index 3938d68de..000000000 --- a/numpy/sys.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from typing import Any - -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... diff --git a/numpy/tests/typing/reveal/modules.py b/numpy/tests/typing/reveal/modules.py index b4a94e326..ca7b14f25 100644 --- a/numpy/tests/typing/reveal/modules.py +++ b/numpy/tests/typing/reveal/modules.py @@ -17,7 +17,6 @@ reveal_type(np.os) # E: ModuleType reveal_type(np.polynomial) # E: ModuleType reveal_type(np.random) # E: ModuleType reveal_type(np.rec) # E: ModuleType -reveal_type(np.sys) # E: ModuleType reveal_type(np.testing) # E: ModuleType reveal_type(np.version) # E: ModuleType reveal_type(np.warnings) # E: ModuleType -- cgit v1.2.1 From e91e31eab3f6a4af1498635c6c80d4ef4263e1d9 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 19 Aug 2020 15:45:17 +0200 Subject: REV: Removed two more (accidently included) builtin modules --- numpy/os.pyi | 4 ---- numpy/tests/typing/reveal/modules.py | 2 -- numpy/warnings.pyi | 4 ---- 3 files changed, 10 deletions(-) delete mode 100644 numpy/os.pyi delete mode 100644 numpy/warnings.pyi diff --git a/numpy/os.pyi b/numpy/os.pyi deleted file mode 100644 index 3938d68de..000000000 --- a/numpy/os.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from typing import Any - -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... diff --git a/numpy/tests/typing/reveal/modules.py b/numpy/tests/typing/reveal/modules.py index ca7b14f25..39eb2dbcb 100644 --- a/numpy/tests/typing/reveal/modules.py +++ b/numpy/tests/typing/reveal/modules.py @@ -13,13 +13,11 @@ reveal_type(np.linalg) # E: ModuleType reveal_type(np.ma) # E: ModuleType reveal_type(np.math) # E: ModuleType reveal_type(np.matrixlib) # E: ModuleType -reveal_type(np.os) # E: ModuleType reveal_type(np.polynomial) # E: ModuleType reveal_type(np.random) # E: ModuleType reveal_type(np.rec) # E: ModuleType reveal_type(np.testing) # E: ModuleType reveal_type(np.version) # E: ModuleType -reveal_type(np.warnings) # E: ModuleType # TODO: Remove when annotations have been added to `np.linalg.norm` reveal_type(np.linalg.norm) # E: Any diff --git a/numpy/warnings.pyi b/numpy/warnings.pyi deleted file mode 100644 index 3938d68de..000000000 --- a/numpy/warnings.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from typing import Any - -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... -- cgit v1.2.1 From 160a782381d2f723b0f1d836503e13eedeaaf7aa Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 19 Aug 2020 15:49:57 +0200 Subject: REV: Removed one last builtin module --- numpy/__init__.pyi | 3 --- numpy/math.pyi | 4 ---- numpy/tests/typing/reveal/modules.py | 1 - 3 files changed, 8 deletions(-) delete mode 100644 numpy/math.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 005a15613..6c32bd2f6 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -55,15 +55,12 @@ from . import ( lib, linalg, ma, - math, matrixlib, - os, polynomial, random, rec, testing, version, - warnings, ) # TODO: remove when the full numpy namespace is defined diff --git a/numpy/math.pyi b/numpy/math.pyi deleted file mode 100644 index 3938d68de..000000000 --- a/numpy/math.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from typing import Any - -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... diff --git a/numpy/tests/typing/reveal/modules.py b/numpy/tests/typing/reveal/modules.py index 39eb2dbcb..736aae10c 100644 --- a/numpy/tests/typing/reveal/modules.py +++ b/numpy/tests/typing/reveal/modules.py @@ -11,7 +11,6 @@ reveal_type(np.fft) # E: ModuleType reveal_type(np.lib) # E: ModuleType reveal_type(np.linalg) # E: ModuleType reveal_type(np.ma) # E: ModuleType -reveal_type(np.math) # E: ModuleType reveal_type(np.matrixlib) # E: ModuleType reveal_type(np.polynomial) # E: ModuleType reveal_type(np.random) # E: ModuleType -- cgit v1.2.1 From 8c226bf9b19a3b974bc63421918d068660ba506c Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 11 Aug 2020 14:27:48 -0600 Subject: MAINT: Remove uses of PyString_FromString. We no longer need to use the compatibility function after dropping support for Python 2.7. In some cases unicode was the correct string type rather than the bytes of the compatibility version and bugs in the array `__complex__` and array `__array_interface__` methods have been fixed by changing that. --- .../upcoming_changes/17068.compatibility.rst | 4 +++ numpy/core/src/multiarray/common.c | 2 +- numpy/core/src/multiarray/getset.c | 2 +- numpy/core/src/multiarray/methods.c | 35 +++++----------------- numpy/core/src/umath/_umath_tests.c.src | 2 +- numpy/f2py/cfuncs.py | 4 +-- numpy/f2py/rules.py | 2 +- numpy/f2py/src/test/foomodule.c | 2 +- numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c | 21 +++++-------- numpy/linalg/umath_linalg.c.src | 2 +- 10 files changed, 28 insertions(+), 48 deletions(-) create mode 100644 doc/release/upcoming_changes/17068.compatibility.rst diff --git a/doc/release/upcoming_changes/17068.compatibility.rst b/doc/release/upcoming_changes/17068.compatibility.rst new file mode 100644 index 000000000..7aa4e58ae --- /dev/null +++ b/doc/release/upcoming_changes/17068.compatibility.rst @@ -0,0 +1,4 @@ +f2py generated code may return unicode instead of byte strings +-------------------------------------------------------------- +Some byte strings previously returned by f2py generated code may now be unicode +strings. This results from the ongoing Python2 -> Python3 cleanup. diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 2abc79167..49ef9e0e7 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -133,7 +133,7 @@ NPY_NO_EXPORT PyArray_Descr * _array_typedescr_fromstr(char const *c_str) { PyArray_Descr *descr = NULL; - PyObject *stringobj = PyString_FromString(c_str); + PyObject *stringobj = PyBytes_FromString(c_str); if (stringobj == NULL) { return NULL; diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 9066f52a8..4ac126927 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -217,7 +217,7 @@ array_protocol_descr_get(PyArrayObject *self) if (dobj == NULL) { return NULL; } - PyTuple_SET_ITEM(dobj, 0, PyString_FromString("")); + PyTuple_SET_ITEM(dobj, 0, PyUnicode_FromString("")); PyTuple_SET_ITEM(dobj, 1, array_typestr_get(self)); res = PyList_New(1); if (res == NULL) { diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index a2db8042f..3270e268b 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -2585,9 +2585,10 @@ array_complex(PyArrayObject *self, PyObject *NPY_UNUSED(args)) PyArrayObject *arr; PyArray_Descr *dtype; PyObject *c; + if (PyArray_SIZE(self) != 1) { - PyErr_SetString(PyExc_TypeError, "only length-1 arrays can "\ - "be converted to Python scalars"); + PyErr_SetString(PyExc_TypeError, + "only length-1 arrays can be converted to Python scalars"); return NULL; } @@ -2598,38 +2599,18 @@ array_complex(PyArrayObject *self, PyObject *NPY_UNUSED(args)) if (!PyArray_CanCastArrayTo(self, dtype, NPY_SAME_KIND_CASTING) && !(PyArray_TYPE(self) == NPY_OBJECT)) { - PyObject *err, *msg_part; + PyObject *descr = (PyObject*)PyArray_DESCR(self); + Py_DECREF(dtype); - err = PyString_FromString("unable to convert "); - if (err == NULL) { - return NULL; - } - msg_part = PyObject_Repr((PyObject*)PyArray_DESCR(self)); - if (msg_part == NULL) { - Py_DECREF(err); - return NULL; - } - PyString_ConcatAndDel(&err, msg_part); - if (err == NULL) { - return NULL; - } - msg_part = PyString_FromString(", to complex."); - if (msg_part == NULL) { - Py_DECREF(err); - return NULL; - } - PyString_ConcatAndDel(&err, msg_part); - if (err == NULL) { - return NULL; - } - PyErr_SetObject(PyExc_TypeError, err); - Py_DECREF(err); + PyErr_Format(PyExc_TypeError, + "Unable to convert %R to complex", descr); return NULL; } if (PyArray_TYPE(self) == NPY_OBJECT) { /* let python try calling __complex__ on the object. */ PyObject *args, *res; + Py_DECREF(dtype); args = Py_BuildValue("(O)", *((PyObject**)PyArray_DATA(self))); if (args == NULL) { diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index d08aabd64..932c3b5ab 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -671,7 +671,7 @@ PyMODINIT_FUNC PyInit__umath_tests(void) { d = PyModule_GetDict(m); - version = PyString_FromString("0.1"); + version = PyUnicode_FromString("0.1"); PyDict_SetItemString(d, "__version__", version); Py_DECREF(version); diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index ccbc9b0fb..9f5c73a45 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -320,10 +320,10 @@ cppmacros[ 'pyobj_from_complex_float1'] = '#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))' needs['pyobj_from_string1'] = ['string'] cppmacros[ - 'pyobj_from_string1'] = '#define pyobj_from_string1(v) (PyString_FromString((char *)v))' + 'pyobj_from_string1'] = '#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))' needs['pyobj_from_string1size'] = ['string'] cppmacros[ - 'pyobj_from_string1size'] = '#define pyobj_from_string1size(v,len) (PyUString_FromStringAndSize((char *)v, len))' + 'pyobj_from_string1size'] = '#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))' needs['TRYPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] cppmacros['TRYPYARRAYTEMPLATE'] = """\ /* New SciPy */ diff --git a/numpy/f2py/rules.py b/numpy/f2py/rules.py index 56f2033ff..a14f60194 100755 --- a/numpy/f2py/rules.py +++ b/numpy/f2py/rules.py @@ -202,7 +202,7 @@ PyMODINIT_FUNC PyInit_#modulename#(void) { \tif (PyErr_Occurred()) \t\t{PyErr_SetString(PyExc_ImportError, \"can't initialize module #modulename# (failed to import numpy)\"); return m;} \td = PyModule_GetDict(m); -\ts = PyString_FromString(\"$R""" + """evision: $\"); +\ts = PyUnicode_FromString(\"$R""" + """evision: $\"); \tPyDict_SetItemString(d, \"__version__\", s); \tPy_DECREF(s); \ts = PyUnicode_FromString( diff --git a/numpy/f2py/src/test/foomodule.c b/numpy/f2py/src/test/foomodule.c index caf3590d4..88ec62440 100644 --- a/numpy/f2py/src/test/foomodule.c +++ b/numpy/f2py/src/test/foomodule.c @@ -121,7 +121,7 @@ void initfoo() { m = Py_InitModule("foo", foo_module_methods); d = PyModule_GetDict(m); - s = PyString_FromString("This module 'foo' demonstrates the usage of fortranobject."); + s = PyUnicode_FromString("This module 'foo' demonstrates the usage of fortranobject."); PyDict_SetItemString(d, "__doc__", s); /* Fortran objects: */ diff --git a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c index 0db33e714..3dadc137d 100644 --- a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c +++ b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c @@ -1,14 +1,9 @@ -/* File: wrapmodule.c - * This file is auto-generated with f2py (version:2_1330). - * Hand edited by Pearu. - * f2py is a Fortran to Python Interface Generator (FPIG), Second Edition, - * written by Pearu Peterson . - * See http://cens.ioc.ee/projects/f2py2e/ - * Generation date: Fri Oct 21 22:41:12 2005 - * $Revision:$ - * $Date:$ - * Do not edit this file directly unless you know what you are doing!!! +/* + * This file was auto-generated with f2py (version:2_1330) and hand edited by + * Pearu for testing purposes. Do not edit this file unless you know what you + * are doing!!! */ + #ifdef __cplusplus extern "C" { #endif @@ -149,9 +144,9 @@ PyMODINIT_FUNC PyInit_test_array_from_pyobj_ext(void) { if (PyErr_Occurred()) Py_FatalError("can't initialize module wrap (failed to import numpy)"); d = PyModule_GetDict(m); - s = PyString_FromString("This module 'wrap' is auto-generated with f2py (version:2_1330).\nFunctions:\n" -" arr = call(type_num,dims,intent,obj)\n" -"."); + s = PyUnicode_FromString("This module 'wrap' is auto-generated with f2py (version:2_1330).\nFunctions:\n" + " arr = call(type_num,dims,intent,obj)\n" + "."); PyDict_SetItemString(d, "__doc__", s); wrap_error = PyErr_NewException ("wrap.error", NULL, NULL); Py_DECREF(s); diff --git a/numpy/linalg/umath_linalg.c.src b/numpy/linalg/umath_linalg.c.src index 59647c67d..1807aadcf 100644 --- a/numpy/linalg/umath_linalg.c.src +++ b/numpy/linalg/umath_linalg.c.src @@ -3665,7 +3665,7 @@ PyObject *PyInit__umath_linalg(void) return NULL; } - version = PyString_FromString(umath_linalg_version_string); + version = PyUnicode_FromString(umath_linalg_version_string); if (version == NULL) { return NULL; } -- cgit v1.2.1 From 68fd054092a0ce41ad013230a2f3aa6aa688175d Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 17:47:56 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Juan Nunez-Iglesias --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 8554cbccd..8f47f8717 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -42,7 +42,7 @@ to that. x = dask.array.arange(5) # Creates dask.array np.diff(x) # Returns dask.array -Note above how we called Dask's implementation of ``sum`` via the NumPy +Note above how we called Dask's implementation of ``diff`` via the NumPy namespace by calling ``np.diff``, and the same would apply if we had a CuPy array or any other array from a library that adopts ``__array_function__``. This allows writing code that is agnostic to the implementation library, thus -- cgit v1.2.1 From 61dcb6382d62408bd07ca708c7ec8a34d734606a Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:12:04 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Juan Nunez-Iglesias --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 8f47f8717..9abbebacd 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -303,7 +303,7 @@ by NEP 37 [6]_, which would require considerable rework by downstream libraries that adopt ``__array_function__`` already, because of that we still believe the ``like=`` argument is beneficial for NumPy and downstream libraries. However, that proposal wouldn't necessarily be considered a direct alternative to the -present NEP, as it would replace NEP 18 entirely, on which this builds upon. +present NEP, as it would replace NEP 18 entirely, upon which this builds. Discussion on details about this new proposal and why that would require rework by downstream libraries is beyond the scopy of the present proposal. -- cgit v1.2.1 From 3cf7b6bb27399c72a6d1bc65a896fcc173a9985d Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:12:46 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Juan Nunez-Iglesias --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 9abbebacd..b023d6fc8 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -305,7 +305,7 @@ that adopt ``__array_function__`` already, because of that we still believe the that proposal wouldn't necessarily be considered a direct alternative to the present NEP, as it would replace NEP 18 entirely, upon which this builds. Discussion on details about this new proposal and why that would require rework -by downstream libraries is beyond the scopy of the present proposal. +by downstream libraries is beyond the scope of the present proposal. Discussion ---------- -- cgit v1.2.1 From 52d9c7455a78e72820cd4bae996646bf9f1c1095 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:18:12 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Chunlin --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index b023d6fc8..e09875d15 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -108,7 +108,7 @@ CuPy arrays: Note in the ``my_pad`` function above how ``arr`` is used as a reference to dictate what array type padding should have, before concatenating the arrays to produce the result. On the other hand, if ``like=`` wasn't used, the NumPy case -case would still work, but CuPy wouldn't allow this kind of automatic +would still work, but CuPy wouldn't allow this kind of automatic conversion, ultimately raising a ``TypeError: Only cupy arrays can be concatenated`` exception. -- cgit v1.2.1 From 615f19ffe653829ed4709cae65d66c0b7b7c2e2e Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:18:24 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Chunlin --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index e09875d15..d0334967c 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -162,7 +162,7 @@ To enable proper identification of the array type we use Dask's utility function function is primarily targeted at the library's internal usage to ensure chunks are created with correct types. Without the ``like=`` argument, it would be impossible to ensure ``my_pad`` creates a padding array with a type matching -that of the input array, which would cause cause a ``TypeError`` exception to +that of the input array, which would cause a ``TypeError`` exception to be raised by CuPy, as discussed above would happen to the CuPy case alone. Current NumPy users who don't use other arrays from downstream libraries should -- cgit v1.2.1 From 69e3e7186a39abeead5adf8edba5a24d71e96cb9 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 09:24:56 -0700 Subject: NEP: Improve NEP-35 abstract per @mattip's suggestion --- .../nep-0035-array-creation-dispatch-with-array-function.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index d0334967c..7b602c15a 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -15,11 +15,10 @@ Abstract -------- We propose the introduction of a new keyword argument ``like=`` to all array -creation functions, this argument permits the creation of an array based on -a non-NumPy reference array passed via that argument, resulting in an array -defined by the downstream library implementing that type, which also implements -the ``__array_function__`` protocol. With this we address one of that -protocol's shortcomings, as described by NEP 18 [1]_. +creation functions to address one of the shortcomings of ``__array_function__``, +as described by NEP 18 [1]_. The ``like=`` keyword argument will create an +instance of the argument's type, enabling direct creation of non-NumPy arrays. +The target array type must implement the ``__array_function__`` protocol. Motivation and Scope -------------------- -- cgit v1.2.1 From cde3543863a711ae211138f045eeebc10e473974 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:26:54 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Matti Picus --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 7b602c15a..4b6147ca8 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -23,7 +23,7 @@ The target array type must implement the ``__array_function__`` protocol. Motivation and Scope -------------------- -Many are the libraries implementing the NumPy API, such as Dask for graph +Many libraries implement the NumPy API, such as Dask for graph computing, CuPy for GPGPU computing, xarray for N-D labeled arrays, etc. All the libraries mentioned have yet another thing in common: they have also adopted the ``__array_function__`` protocol; a protocol that allows NumPy to understand -- cgit v1.2.1 From 101700710dbe5bdfa891d43f1fc14fb6439082ea Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:27:23 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Matti Picus --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 4b6147ca8..d783df5b0 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -24,9 +24,8 @@ Motivation and Scope -------------------- Many libraries implement the NumPy API, such as Dask for graph -computing, CuPy for GPGPU computing, xarray for N-D labeled arrays, etc. All -the libraries mentioned have yet another thing in common: they have also adopted -the ``__array_function__`` protocol; a protocol that allows NumPy to understand +computing, CuPy for GPGPU computing, xarray for N-D labeled arrays, etc. Underneath, +they have adopted the ``__array_function__`` protocol which allows NumPy to understand and treat downstream objects as if they are the native ``numpy.ndarray`` object. Hence the community while using various libraries still benefits from a unified NumPy API. This not only brings great convenience for standardization but also -- cgit v1.2.1 From a82cc4b5cb4c3926d2b555d9e8da1146c472a17c Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 09:41:38 -0700 Subject: NEP: Move NumPy users comment to top of NEP-35 Usage and Impact --- ...nep-0035-array-creation-dispatch-with-array-function.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index d783df5b0..9fb117005 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -87,6 +87,12 @@ of the keyword-only argument standard described in PEP-3102 [2]_ to implement Usage and Impact ---------------- +NumPy users who don't use other arrays from downstream libraries can continue +to use array creation routines without a ``like=`` argument. Using +``like=np.ndarray`` will work as if no array was passed via that argument. +However, this will incur additional checks that will negatively impact +performance. + To understand the intended use for ``like=``, and before we move to more complex cases, consider the following illustrative example consisting only of NumPy and CuPy arrays: @@ -163,13 +169,6 @@ impossible to ensure ``my_pad`` creates a padding array with a type matching that of the input array, which would cause a ``TypeError`` exception to be raised by CuPy, as discussed above would happen to the CuPy case alone. -Current NumPy users who don't use other arrays from downstream libraries should -have no impact in their current usage of the NumPy API. In the event of the -user passing a NumPy array to ``like=``, that will continue to work as if no -array was passed via that argument. However, this is advised against, as -internally there will be additional checks required that will have an impact in -performance. - Backward Compatibility ---------------------- -- cgit v1.2.1 From 17620c2ecb3dc6c2ab3164fa946c373a120d5619 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:44:09 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Eric Wieser --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 9fb117005..c1edfa750 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -52,7 +52,7 @@ elsewhere and let NumPy handle them. But still these arrays have to be started in their native library and brought back. Instead if it was possible to create these objects through NumPy API then there would be an almost complete experience, all using NumPy syntax. For example, say we have some CuPy array -``cp_arr`` , and want a similar CuPy array with identity matrix. We could still +``cp_arr``, and want a similar CuPy array with identity matrix. We could still write the following: .. code:: python -- cgit v1.2.1 From f1d156219ea154a469d6d2b0d0cc4883c53ceba9 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:56:35 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Eric Wieser --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index c1edfa750..e31f975e7 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -124,7 +124,7 @@ strictly on arrays, which are the objects we can use ``__array_function__`` with. Dask uses a graph computing model, meaning it breaks down a large problem in -many smaller problems and merge their results to reach the final result. To +many smaller problems and merges their results to reach the final result. To break the problem down into smaller ones, Dask also breaks arrays into smaller arrays, that it calls "chunks". A Dask array can thus consist of one or more chunks and they may be of different types. However, in the context of -- cgit v1.2.1 From 57d6bab5c5fa7c0e6b4830988b855fb28f257649 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:56:53 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Eric Wieser --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index e31f975e7..615039442 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -126,7 +126,7 @@ with. Dask uses a graph computing model, meaning it breaks down a large problem in many smaller problems and merges their results to reach the final result. To break the problem down into smaller ones, Dask also breaks arrays into smaller -arrays, that it calls "chunks". A Dask array can thus consist of one or more +arrays that it calls "chunks". A Dask array can thus consist of one or more chunks and they may be of different types. However, in the context of ``__array_function__``, Dask only allows chunks of the same type, for example, a Dask array can be formed of several NumPy arrays or several CuPy arrays, but -- cgit v1.2.1 From 67c973306a445846a3c14e0cf66608be9d89a093 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:57:21 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Eric Wieser --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 615039442..c5a266dfe 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -128,7 +128,7 @@ many smaller problems and merges their results to reach the final result. To break the problem down into smaller ones, Dask also breaks arrays into smaller arrays that it calls "chunks". A Dask array can thus consist of one or more chunks and they may be of different types. However, in the context of -``__array_function__``, Dask only allows chunks of the same type, for example, +``__array_function__``, Dask only allows chunks of the same type; for example, a Dask array can be formed of several NumPy arrays or several CuPy arrays, but not a mix of both. -- cgit v1.2.1 From 974c02378b3b240f370755c0838eefaa422b28ba Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 18:57:41 +0200 Subject: Update doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst Co-authored-by: Eric Wieser --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index c5a266dfe..4ed00d15e 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -132,9 +132,9 @@ chunks and they may be of different types. However, in the context of a Dask array can be formed of several NumPy arrays or several CuPy arrays, but not a mix of both. -To avoid mismatched types during compute, Dask keeps an attribute ``_meta`` as -part of its array throughout computation, this attribute is used to both predict -the output type at graph creation time and to create any intermediary arrays +To avoid mismatched types during computation, Dask keeps an attribute ``_meta`` as +part of its array throughout computation: this attribute is used to both predict +the output type at graph creation time, and to create any intermediary arrays that are necessary within some function's computation. Going back to our previous example, we can use ``_meta`` information to identify what kind of array we would use for padding, as seen below: -- cgit v1.2.1 From b5f55775e4518a0356bf3e4c068e96beccb72b36 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 10:02:53 -0700 Subject: NEP: Clarify NEP-35 C implementation details. --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 4ed00d15e..56ae8600b 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -288,9 +288,11 @@ There are two downsides to the implementation above for C functions: 2. To follow current implementation standards, documentation should be attached directly to the Python source code. -The first version of this proposal suggested the C implementation above as one -viable solution. However, due to the downsides pointed above we have decided to -implement that entirely in C. Please refer to [implementation]_ for details. +The first version of this proposal suggested the implementation above as one +viable solution for NumPy functions implemented in C. However, due to the +downsides pointed above we have decided to discard any changes on the Python +side and resolve those issues with a pure-C implementation . Please refer to +[implementation]_ for details. Alternatives ------------ -- cgit v1.2.1 From 2e30534564aaf49cb0a12baf6b721b7b31402647 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 10:41:45 -0700 Subject: NEP: Clarify Dask intent with `my_dask_pad` function name --- ...nep-0035-array-creation-dispatch-with-array-function.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 56ae8600b..7acb396f6 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -146,19 +146,22 @@ array we would use for padding, as seen below: import dask.array as da from dask.array.utils import meta_from_array - def my_pad(arr, padding): + def my_dask_pad(arr, padding): padding = np.array(padding, like=meta_from_array(arr)) return np.concatenate((padding, arr, padding)) # Returns dask.array - my_pad(da.arange(5), [-1, -1]) + my_dask_pad(da.arange(5), [-1, -1]) # Returns dask.array - my_pad(da.from_array(cupy.arange(5)), [-1, -1]) + my_dask_pad(da.from_array(cupy.arange(5)), [-1, -1]) Note how ``chunktype`` in the return value above changes from -``numpy.ndarray`` in the first ``my_pad`` call to ``cupy.ndarray`` in the -second. +``numpy.ndarray`` in the first ``my_dask_pad`` call to ``cupy.ndarray`` in the +second. We have also renamed the function to ``my_dask_pad`` in this example +with the intent to make it clear that this is how Dask would implement such +functionality, should it need to do so, as it requires Dask's internal tools +that are not of much use elsewhere. To enable proper identification of the array type we use Dask's utility function ``meta_from_array``, which was introduced as part of the work to support -- cgit v1.2.1 From 3d527eacf5064b4ab7dcd4c20de952fe37d931d5 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 10:50:15 -0700 Subject: NEP: Improve grammar on NEP-35 reference to Dask's objects --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 7acb396f6..d7d670839 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -118,10 +118,10 @@ conversion, ultimately raising a Now we should look at how a library like Dask could benefit from ``like=``. Before we understand that, it's important to understand a bit about Dask basics -and ensures correctness with ``__array_function__``. Note that Dask can compute -different sorts of objects, like dataframes, bags and arrays, here we will focus -strictly on arrays, which are the objects we can use ``__array_function__`` -with. +and ensures correctness with ``__array_function__``. Note that Dask can perform +computations on different sorts of objects, like dataframes, bags and arrays, +here we will focus strictly on arrays, which are the objects we can use +``__array_function__`` with. Dask uses a graph computing model, meaning it breaks down a large problem in many smaller problems and merges their results to reach the final result. To -- cgit v1.2.1 From b6f2c16e23be6fdd38ee89233d13ceecd595c948 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 10:53:04 -0700 Subject: NEP: Fix some grammar and formatting in NEP-35 --- doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index d7d670839..cb44a4545 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -293,8 +293,8 @@ There are two downsides to the implementation above for C functions: The first version of this proposal suggested the implementation above as one viable solution for NumPy functions implemented in C. However, due to the -downsides pointed above we have decided to discard any changes on the Python -side and resolve those issues with a pure-C implementation . Please refer to +downsides pointed out above we have decided to discard any changes on the Python +side and resolve those issues with a pure-C implementation. Please refer to [implementation]_ for details. Alternatives -- cgit v1.2.1 From 434d2f141d7e473040effdba37e29ffd3b75a25c Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Wed, 19 Aug 2020 10:02:33 -0600 Subject: TST: Add tests for bugs fixed in gh-17068. - Invalid conversion error bug. - ``__array_interface__['descr']`` bug --- numpy/core/tests/test_regression.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 51cf7039f..f778d4d7c 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -2488,3 +2488,16 @@ class TestRegression: assert arr.size * arr.itemsize > 2 ** 31 c_arr = np.ctypeslib.as_ctypes(arr) assert_equal(c_arr._length_, arr.size) + + def test_complex_conversion_error(self): + # gh-17068 + with pytest.raises(TypeError, match=r"Unable to convert dtype.*"): + complex(np.array("now", np.datetime64)) + + def test__array_interface__descr(self): + # gh-17068 + dt = np.dtype(dict(names=['a', 'b'], + offsets=[0, 0], + formats=[np.int64, np.int64])) + descr = np.array((1, 1), dtype=dt).__array_interface__['descr'] + assert descr == [('', '|V8')] # instead of [(b'', '|V8')] -- cgit v1.2.1 From 4af8fd39bfa4a30e9d26e8021b42ffadf1f99398 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Wed, 19 Aug 2020 20:51:09 +0200 Subject: ENH: Replace module-level `__getattr__` with explicitly defined objects --- numpy/char.pyi | 53 ++++- numpy/compat/__init__.pyi | 26 ++- numpy/core/__init__.pyi | 425 ++++++++++++++++++++++++++++++++++- numpy/ctypeslib.pyi | 8 +- numpy/emath.pyi | 11 +- numpy/lib/__init__.pyi | 177 ++++++++++++++- numpy/ma/__init__.pyi | 225 ++++++++++++++++++- numpy/matrixlib/__init__.pyi | 6 +- numpy/random/__init__.pyi | 61 ++++- numpy/rec.pyi | 5 +- numpy/testing/__init__.pyi | 44 +++- numpy/tests/typing/fail/modules.py | 3 + numpy/tests/typing/reveal/modules.py | 4 +- 13 files changed, 1024 insertions(+), 24 deletions(-) create mode 100644 numpy/tests/typing/fail/modules.py diff --git a/numpy/char.pyi b/numpy/char.pyi index 3938d68de..0e7342c0b 100644 --- a/numpy/char.pyi +++ b/numpy/char.pyi @@ -1,4 +1,53 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +equal: Any +not_equal: Any +greater_equal: Any +less_equal: Any +greater: Any +less: Any +str_len: Any +add: Any +multiply: Any +mod: Any +capitalize: Any +center: Any +count: Any +decode: Any +encode: Any +endswith: Any +expandtabs: Any +find: Any +index: Any +isalnum: Any +isalpha: Any +isdigit: Any +islower: Any +isspace: Any +istitle: Any +isupper: Any +join: Any +ljust: Any +lower: Any +lstrip: Any +partition: Any +replace: Any +rfind: Any +rindex: Any +rjust: Any +rpartition: Any +rsplit: Any +rstrip: Any +split: Any +splitlines: Any +startswith: Any +strip: Any +swapcase: Any +title: Any +translate: Any +upper: Any +zfill: Any +isnumeric: Any +isdecimal: Any +array: Any +asarray: Any diff --git a/numpy/compat/__init__.pyi b/numpy/compat/__init__.pyi index 3938d68de..ff55ccfde 100644 --- a/numpy/compat/__init__.pyi +++ b/numpy/compat/__init__.pyi @@ -1,4 +1,26 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +getargspec: Any +formatargspec: Any +bytes: Any +asbytes: Any +isfileobj: Any +getexception: Any +strchar: Any +unicode: Any +asunicode: Any +asbytes_nested: Any +asunicode_nested: Any +asstr: Any +open_latin1: Any +long: Any +basestring: Any +sixu: Any +integer_types: Any +is_pathlib_path: Any +npy_load_module: Any +Path: Any +pickle: Any +contextlib_nullcontext: Any +os_fspath: Any +os_PathLike: Any diff --git a/numpy/core/__init__.pyi b/numpy/core/__init__.pyi index 3938d68de..9e3cf3d75 100644 --- a/numpy/core/__init__.pyi +++ b/numpy/core/__init__.pyi @@ -1,4 +1,425 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +char: Any +rec: Any +memmap: Any +newaxis: Any +ndarray: Any +flatiter: Any +nditer: Any +nested_iters: Any +ufunc: Any +arange: Any +array: Any +zeros: Any +count_nonzero: Any +empty: Any +broadcast: Any +dtype: Any +fromstring: Any +fromfile: Any +frombuffer: Any +where: Any +argwhere: Any +copyto: Any +concatenate: Any +fastCopyAndTranspose: Any +lexsort: Any +set_numeric_ops: Any +can_cast: Any +promote_types: Any +min_scalar_type: Any +result_type: Any +isfortran: Any +empty_like: Any +zeros_like: Any +ones_like: Any +correlate: Any +convolve: Any +inner: Any +dot: Any +outer: Any +vdot: Any +roll: Any +rollaxis: Any +moveaxis: Any +cross: Any +tensordot: Any +little_endian: Any +fromiter: Any +array_equal: Any +array_equiv: Any +indices: Any +fromfunction: Any +isclose: Any +isscalar: Any +binary_repr: Any +base_repr: Any +ones: Any +identity: Any +allclose: Any +compare_chararrays: Any +putmask: Any +flatnonzero: Any +Inf: Any +inf: Any +infty: Any +Infinity: Any +nan: Any +NaN: Any +False_: Any +True_: Any +bitwise_not: Any +CLIP: Any +RAISE: Any +WRAP: Any +MAXDIMS: Any +BUFSIZE: Any +ALLOW_THREADS: Any +ComplexWarning: Any +full: Any +full_like: Any +matmul: Any +shares_memory: Any +may_share_memory: Any +MAY_SHARE_BOUNDS: Any +MAY_SHARE_EXACT: Any +TooHardError: Any +AxisError: Any +alen: Any +all: Any +alltrue: Any +amax: Any +amin: Any +any: Any +argmax: Any +argmin: Any +argpartition: Any +argsort: Any +around: Any +choose: Any +clip: Any +compress: Any +cumprod: Any +cumproduct: Any +cumsum: Any +diagonal: Any +mean: Any +ndim: Any +nonzero: Any +partition: Any +prod: Any +product: Any +ptp: Any +put: Any +ravel: Any +repeat: Any +reshape: Any +resize: Any +round_: Any +searchsorted: Any +shape: Any +size: Any +sometrue: Any +sort: Any +squeeze: Any +std: Any +sum: Any +swapaxes: Any +take: Any +trace: Any +transpose: Any +var: Any +_UFUNC_API: Any +ERR_CALL: Any +ERR_DEFAULT: Any +ERR_IGNORE: Any +ERR_LOG: Any +ERR_PRINT: Any +ERR_RAISE: Any +ERR_WARN: Any +FLOATING_POINT_SUPPORT: Any +FPE_DIVIDEBYZERO: Any +FPE_INVALID: Any +FPE_OVERFLOW: Any +FPE_UNDERFLOW: Any +NAN: Any +NINF: Any +NZERO: Any +PINF: Any +PZERO: Any +SHIFT_DIVIDEBYZERO: Any +SHIFT_INVALID: Any +SHIFT_OVERFLOW: Any +SHIFT_UNDERFLOW: Any +UFUNC_BUFSIZE_DEFAULT: Any +UFUNC_PYVALS_NAME: Any +_add_newdoc_ufunc: Any +absolute: Any +add: Any +arccos: Any +arccosh: Any +arcsin: Any +arcsinh: Any +arctan: Any +arctan2: Any +arctanh: Any +bitwise_and: Any +bitwise_or: Any +bitwise_xor: Any +cbrt: Any +ceil: Any +conj: Any +conjugate: Any +copysign: Any +cos: Any +cosh: Any +deg2rad: Any +degrees: Any +divide: Any +divmod: Any +e: Any +equal: Any +euler_gamma: Any +exp: Any +exp2: Any +expm1: Any +fabs: Any +floor: Any +floor_divide: Any +float_power: Any +fmax: Any +fmin: Any +fmod: Any +frexp: Any +frompyfunc: Any +gcd: Any +geterrobj: Any +greater: Any +greater_equal: Any +heaviside: Any +hypot: Any +invert: Any +isfinite: Any +isinf: Any +isnan: Any +isnat: Any +lcm: Any +ldexp: Any +left_shift: Any +less: Any +less_equal: Any +log: Any +log10: Any +log1p: Any +log2: Any +logaddexp: Any +logaddexp2: Any +logical_and: Any +logical_not: Any +logical_or: Any +logical_xor: Any +maximum: Any +minimum: Any +mod: Any +modf: Any +multiply: Any +negative: Any +nextafter: Any +not_equal: Any +pi: Any +positive: Any +power: Any +rad2deg: Any +radians: Any +reciprocal: Any +remainder: Any +right_shift: Any +rint: Any +seterrobj: Any +sign: Any +signbit: Any +sin: Any +sinh: Any +spacing: Any +sqrt: Any +square: Any +subtract: Any +tan: Any +tanh: Any +true_divide: Any +trunc: Any +sctypeDict: Any +typeDict: Any +sctypes: Any +ScalarType: Any +obj2sctype: Any +cast: Any +nbytes: Any +sctype2char: Any +maximum_sctype: Any +issctype: Any +typecodes: Any +find_common_type: Any +issubdtype: Any +datetime_data: Any +datetime_as_string: Any +busday_offset: Any +busday_count: Any +is_busday: Any +busdaycalendar: Any +byte: Any +ubyte: Any +short: Any +ushort: Any +uint: Any +intp: Any +uintp: Any +long: Any +longlong: Any +ulonglong: Any +half: Any +double: Any +longdouble: Any +cfloat: Any +cdouble: Any +clongdouble: Any +unicode: Any +void: Any +generic: Any +number: Any +integer: Any +inexact: Any +signedinteger: Any +unsignedinteger: Any +floating: Any +complexfloating: Any +flexible: Any +character: Any +bool8: Any +int64: Any +uint64: Any +float16: Any +float32: Any +float64: Any +complex64: Any +complex128: Any +object0: Any +bytes0: Any +str0: Any +void0: Any +datetime64: Any +timedelta64: Any +Bytes0: Any +Datetime64: Any +Str0: Any +Uint64: Any +int32: Any +uint32: Any +int16: Any +uint16: Any +int8: Any +uint8: Any +complex_: Any +int0: Any +uint0: Any +single: Any +csingle: Any +singlecomplex: Any +float_: Any +intc: Any +uintc: Any +int_: Any +longfloat: Any +clongfloat: Any +longcomplex: Any +bool_: Any +bytes_: Any +string_: Any +str_: Any +unicode_: Any +object_: Any +array2string: Any +array_str: Any +array_repr: Any +set_string_function: Any +set_printoptions: Any +get_printoptions: Any +printoptions: Any +format_float_positional: Any +format_float_scientific: Any +asarray: Any +asanyarray: Any +ascontiguousarray: Any +asfortranarray: Any +require: Any +seterr: Any +geterr: Any +setbufsize: Any +getbufsize: Any +seterrcall: Any +geterrcall: Any +errstate: Any +alen: Any +all: Any +alltrue: Any +amax: Any +amin: Any +any: Any +argmax: Any +argmin: Any +argpartition: Any +argsort: Any +around: Any +choose: Any +clip: Any +compress: Any +cumprod: Any +cumproduct: Any +cumsum: Any +diagonal: Any +mean: Any +ndim: Any +nonzero: Any +partition: Any +prod: Any +product: Any +ptp: Any +put: Any +ravel: Any +repeat: Any +reshape: Any +resize: Any +round_: Any +searchsorted: Any +shape: Any +size: Any +sometrue: Any +sort: Any +squeeze: Any +std: Any +sum: Any +swapaxes: Any +take: Any +trace: Any +transpose: Any +var: Any +record: Any +recarray: Any +format_parser: Any +chararray: Any +logspace: Any +linspace: Any +geomspace: Any +MachAr: Any +finfo: Any +iinfo: Any +atleast_1d: Any +atleast_2d: Any +atleast_3d: Any +block: Any +hstack: Any +stack: Any +vstack: Any +einsum: Any +einsum_path: Any diff --git a/numpy/ctypeslib.pyi b/numpy/ctypeslib.pyi index 3938d68de..c71d2dda2 100644 --- a/numpy/ctypeslib.pyi +++ b/numpy/ctypeslib.pyi @@ -1,4 +1,8 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +load_library: Any +ndpointer: Any +ctypes_load_library: Any +c_intp: Any +as_ctypes: Any +as_array: Any diff --git a/numpy/emath.pyi b/numpy/emath.pyi index 3938d68de..032ec9505 100644 --- a/numpy/emath.pyi +++ b/numpy/emath.pyi @@ -1,4 +1,11 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +sqrt: Any +log: Any +log2: Any +logn: Any +log10: Any +power: Any +arccos: Any +arcsin: Any +arctanh: Any diff --git a/numpy/lib/__init__.pyi b/numpy/lib/__init__.pyi index 3938d68de..413e2ae1b 100644 --- a/numpy/lib/__init__.pyi +++ b/numpy/lib/__init__.pyi @@ -1,4 +1,177 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +emath: Any +math: Any +tracemalloc_domain: Any +Arrayterator: Any +iscomplexobj: Any +isrealobj: Any +imag: Any +iscomplex: Any +isreal: Any +nan_to_num: Any +real: Any +real_if_close: Any +typename: Any +asfarray: Any +mintypecode: Any +asscalar: Any +common_type: Any +ravel_multi_index: Any +unravel_index: Any +mgrid: Any +ogrid: Any +r_: Any +c_: Any +s_: Any +index_exp: Any +ix_: Any +ndenumerate: Any +ndindex: Any +fill_diagonal: Any +diag_indices: Any +diag_indices_from: Any +select: Any +piecewise: Any +trim_zeros: Any +copy: Any +iterable: Any +percentile: Any +diff: Any +gradient: Any +angle: Any +unwrap: Any +sort_complex: Any +disp: Any +flip: Any +rot90: Any +extract: Any +place: Any +vectorize: Any +asarray_chkfinite: Any +average: Any +bincount: Any +digitize: Any +cov: Any +corrcoef: Any +msort: Any +median: Any +sinc: Any +hamming: Any +hanning: Any +bartlett: Any +blackman: Any +kaiser: Any +trapz: Any +i0: Any +add_newdoc: Any +add_docstring: Any +meshgrid: Any +delete: Any +insert: Any +append: Any +interp: Any +add_newdoc_ufunc: Any +quantile: Any +column_stack: Any +row_stack: Any +dstack: Any +array_split: Any +split: Any +hsplit: Any +vsplit: Any +dsplit: Any +apply_over_axes: Any +expand_dims: Any +apply_along_axis: Any +kron: Any +tile: Any +get_array_wrap: Any +take_along_axis: Any +put_along_axis: Any +broadcast_to: Any +broadcast_arrays: Any +diag: Any +diagflat: Any +eye: Any +fliplr: Any +flipud: Any +tri: Any +triu: Any +tril: Any +vander: Any +histogram2d: Any +mask_indices: Any +tril_indices: Any +tril_indices_from: Any +triu_indices: Any +triu_indices_from: Any +fix: Any +isneginf: Any +isposinf: Any +pad: Any +poly: Any +roots: Any +polyint: Any +polyder: Any +polyadd: Any +polysub: Any +polymul: Any +polydiv: Any +polyval: Any +poly1d: Any +polyfit: Any +RankWarning: Any +issubclass_: Any +issubsctype: Any +issubdtype: Any +deprecate: Any +deprecate_with_doc: Any +get_include: Any +info: Any +source: Any +who: Any +lookfor: Any +byte_bounds: Any +safe_eval: Any +ediff1d: Any +intersect1d: Any +setxor1d: Any +union1d: Any +setdiff1d: Any +unique: Any +in1d: Any +isin: Any +savetxt: Any +loadtxt: Any +genfromtxt: Any +ndfromtxt: Any +mafromtxt: Any +recfromtxt: Any +recfromcsv: Any +load: Any +loads: Any +save: Any +savez: Any +savez_compressed: Any +packbits: Any +unpackbits: Any +fromregex: Any +DataSource: Any +nansum: Any +nanmax: Any +nanmin: Any +nanargmax: Any +nanargmin: Any +nanmean: Any +nanmedian: Any +nanpercentile: Any +nanvar: Any +nanstd: Any +nanprod: Any +nancumsum: Any +nancumprod: Any +nanquantile: Any +histogram: Any +histogramdd: Any +histogram_bin_edges: Any diff --git a/numpy/ma/__init__.pyi b/numpy/ma/__init__.pyi index 3938d68de..d1259abcc 100644 --- a/numpy/ma/__init__.pyi +++ b/numpy/ma/__init__.pyi @@ -1,4 +1,225 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +core: Any +extras: Any +MAError: Any +MaskError: Any +MaskType: Any +MaskedArray: Any +abs: Any +absolute: Any +add: Any +all: Any +allclose: Any +allequal: Any +alltrue: Any +amax: Any +amin: Any +angle: Any +anom: Any +anomalies: Any +any: Any +append: Any +arange: Any +arccos: Any +arccosh: Any +arcsin: Any +arcsinh: Any +arctan: Any +arctan2: Any +arctanh: Any +argmax: Any +argmin: Any +argsort: Any +around: Any +array: Any +asanyarray: Any +asarray: Any +bitwise_and: Any +bitwise_or: Any +bitwise_xor: Any +bool_: Any +ceil: Any +choose: Any +clip: Any +common_fill_value: Any +compress: Any +compressed: Any +concatenate: Any +conjugate: Any +convolve: Any +copy: Any +correlate: Any +cos: Any +cosh: Any +count: Any +cumprod: Any +cumsum: Any +default_fill_value: Any +diag: Any +diagonal: Any +diff: Any +divide: Any +empty: Any +empty_like: Any +equal: Any +exp: Any +expand_dims: Any +fabs: Any +filled: Any +fix_invalid: Any +flatten_mask: Any +flatten_structured_array: Any +floor: Any +floor_divide: Any +fmod: Any +frombuffer: Any +fromflex: Any +fromfunction: Any +getdata: Any +getmask: Any +getmaskarray: Any +greater: Any +greater_equal: Any +harden_mask: Any +hypot: Any +identity: Any +ids: Any +indices: Any +inner: Any +innerproduct: Any +isMA: Any +isMaskedArray: Any +is_mask: Any +is_masked: Any +isarray: Any +left_shift: Any +less: Any +less_equal: Any +log: Any +log10: Any +log2: Any +logical_and: Any +logical_not: Any +logical_or: Any +logical_xor: Any +make_mask: Any +make_mask_descr: Any +make_mask_none: Any +mask_or: Any +masked: Any +masked_array: Any +masked_equal: Any +masked_greater: Any +masked_greater_equal: Any +masked_inside: Any +masked_invalid: Any +masked_less: Any +masked_less_equal: Any +masked_not_equal: Any +masked_object: Any +masked_outside: Any +masked_print_option: Any +masked_singleton: Any +masked_values: Any +masked_where: Any +max: Any +maximum: Any +maximum_fill_value: Any +mean: Any +min: Any +minimum: Any +minimum_fill_value: Any +mod: Any +multiply: Any +mvoid: Any +ndim: Any +negative: Any +nomask: Any +nonzero: Any +not_equal: Any +ones: Any +outer: Any +outerproduct: Any +power: Any +prod: Any +product: Any +ptp: Any +put: Any +putmask: Any +ravel: Any +remainder: Any +repeat: Any +reshape: Any +resize: Any +right_shift: Any +round: Any +round_: Any +set_fill_value: Any +shape: Any +sin: Any +sinh: Any +size: Any +soften_mask: Any +sometrue: Any +sort: Any +sqrt: Any +squeeze: Any +std: Any +subtract: Any +sum: Any +swapaxes: Any +take: Any +tan: Any +tanh: Any +trace: Any +transpose: Any +true_divide: Any +var: Any +where: Any +zeros: Any +apply_along_axis: Any +apply_over_axes: Any +atleast_1d: Any +atleast_2d: Any +atleast_3d: Any +average: Any +clump_masked: Any +clump_unmasked: Any +column_stack: Any +compress_cols: Any +compress_nd: Any +compress_rowcols: Any +compress_rows: Any +count_masked: Any +corrcoef: Any +cov: Any +diagflat: Any +dot: Any +dstack: Any +ediff1d: Any +flatnotmasked_contiguous: Any +flatnotmasked_edges: Any +hsplit: Any +hstack: Any +isin: Any +in1d: Any +intersect1d: Any +mask_cols: Any +mask_rowcols: Any +mask_rows: Any +masked_all: Any +masked_all_like: Any +median: Any +mr_: Any +notmasked_contiguous: Any +notmasked_edges: Any +polyfit: Any +row_stack: Any +setdiff1d: Any +setxor1d: Any +stack: Any +unique: Any +union1d: Any +vander: Any +vstack: Any diff --git a/numpy/matrixlib/__init__.pyi b/numpy/matrixlib/__init__.pyi index 3938d68de..b240bb327 100644 --- a/numpy/matrixlib/__init__.pyi +++ b/numpy/matrixlib/__init__.pyi @@ -1,4 +1,6 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +matrix: Any +bmat: Any +mat: Any +asmatrix: Any diff --git a/numpy/random/__init__.pyi b/numpy/random/__init__.pyi index 3938d68de..f7c3cfafe 100644 --- a/numpy/random/__init__.pyi +++ b/numpy/random/__init__.pyi @@ -1,4 +1,61 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +beta: Any +binomial: Any +bytes: Any +chisquare: Any +choice: Any +dirichlet: Any +exponential: Any +f: Any +gamma: Any +geometric: Any +get_state: Any +gumbel: Any +hypergeometric: Any +laplace: Any +logistic: Any +lognormal: Any +logseries: Any +multinomial: Any +multivariate_normal: Any +negative_binomial: Any +noncentral_chisquare: Any +noncentral_f: Any +normal: Any +pareto: Any +permutation: Any +poisson: Any +power: Any +rand: Any +randint: Any +randn: Any +random: Any +random_integers: Any +random_sample: Any +ranf: Any +rayleigh: Any +sample: Any +seed: Any +set_state: Any +shuffle: Any +standard_cauchy: Any +standard_exponential: Any +standard_gamma: Any +standard_normal: Any +standard_t: Any +triangular: Any +uniform: Any +vonmises: Any +wald: Any +weibull: Any +zipf: Any +Generator: Any +RandomState: Any +SeedSequence: Any +MT19937: Any +Philox: Any +PCG64: Any +SFC64: Any +default_rng: Any +BitGenerator: Any diff --git a/numpy/rec.pyi b/numpy/rec.pyi index 3938d68de..c70ee5374 100644 --- a/numpy/rec.pyi +++ b/numpy/rec.pyi @@ -1,4 +1,5 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +record: Any +recarray: Any +format_parser: Any diff --git a/numpy/testing/__init__.pyi b/numpy/testing/__init__.pyi index 3938d68de..c394a387d 100644 --- a/numpy/testing/__init__.pyi +++ b/numpy/testing/__init__.pyi @@ -1,4 +1,44 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +assert_equal: Any +assert_almost_equal: Any +assert_approx_equal: Any +assert_array_equal: Any +assert_array_less: Any +assert_string_equal: Any +assert_array_almost_equal: Any +assert_raises: Any +build_err_msg: Any +decorate_methods: Any +jiffies: Any +memusage: Any +print_assert_equal: Any +raises: Any +rundocs: Any +runstring: Any +verbose: Any +measure: Any +assert_: Any +assert_array_almost_equal_nulp: Any +assert_raises_regex: Any +assert_array_max_ulp: Any +assert_warns: Any +assert_no_warnings: Any +assert_allclose: Any +IgnoreException: Any +clear_and_catch_warnings: Any +SkipTest: Any +KnownFailureException: Any +temppath: Any +tempdir: Any +IS_PYPY: Any +HAS_REFCOUNT: Any +suppress_warnings: Any +assert_array_compare: Any +_assert_valid_refcount: Any +_gen_alignment_data: Any +assert_no_gc_cycles: Any +break_cycles: Any +HAS_LAPACK64: Any +TestCase: Any +run_module_suite: Any diff --git a/numpy/tests/typing/fail/modules.py b/numpy/tests/typing/fail/modules.py new file mode 100644 index 000000000..e7ffe8920 --- /dev/null +++ b/numpy/tests/typing/fail/modules.py @@ -0,0 +1,3 @@ +import numpy as np + +np.testing.bob # E: Module has no attribute diff --git a/numpy/tests/typing/reveal/modules.py b/numpy/tests/typing/reveal/modules.py index 736aae10c..16709496b 100644 --- a/numpy/tests/typing/reveal/modules.py +++ b/numpy/tests/typing/reveal/modules.py @@ -18,5 +18,5 @@ reveal_type(np.rec) # E: ModuleType reveal_type(np.testing) # E: ModuleType reveal_type(np.version) # E: ModuleType -# TODO: Remove when annotations have been added to `np.linalg.norm` -reveal_type(np.linalg.norm) # E: Any +# TODO: Remove when annotations have been added to `np.testing.assert_equal` +reveal_type(np.testing.assert_equal) # E: Any -- cgit v1.2.1 From 57f78df125892eaa4539913759f3ac29ff4aef6a Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Wed, 19 Aug 2020 15:13:05 -0700 Subject: ENH: Clarifies meta_from_array function in NEP-35 --- ...array-creation-dispatch-with-array-function.rst | 32 ++++++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index cb44a4545..dca8b2418 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -56,12 +56,14 @@ experience, all using NumPy syntax. For example, say we have some CuPy array write the following: .. code:: python + x = cupy.identity(3) Instead, the better way would be using to only use the NumPy API, this could now be achieved with: .. code:: python + x = np.identity(3, like=cp_arr) As if by magic, ``x`` will also be a CuPy array, as NumPy was capable to infer @@ -165,12 +167,30 @@ that are not of much use elsewhere. To enable proper identification of the array type we use Dask's utility function ``meta_from_array``, which was introduced as part of the work to support -``__array_function__``, allowing Dask to handle ``_meta`` appropriately. That -function is primarily targeted at the library's internal usage to ensure chunks -are created with correct types. Without the ``like=`` argument, it would be -impossible to ensure ``my_pad`` creates a padding array with a type matching -that of the input array, which would cause a ``TypeError`` exception to -be raised by CuPy, as discussed above would happen to the CuPy case alone. +``__array_function__``, allowing Dask to handle ``_meta`` appropriately. Readers +can think of ``meta_from_array`` as a special function that just returns the +type of the underlying Dask array, for example: + +.. code:: python + + np_arr = da.arange(5) + cp_arr = da.from_array(cupy.arange(5)) + + meta_from_array(np_arr) # Returns a numpy.ndarray + meta_from_array(cp_arr) # Returns a cupy.ndarray + +Since the value returned by ``meta_from_array`` is a NumPy-like array, we can +just pass that directly into the ``like=`` argument. + +The ``meta_from_array`` function is primarily targeted at the library's internal +usage to ensure chunks are created with correct types. Without the ``like=`` +argument, it would be impossible to ensure ``my_pad`` creates a padding array +with a type matching that of the input array, which would cause a ``TypeError`` +exception to be raised by CuPy, as discussed above would happen to the CuPy case +alone. Combining Dask's internal handling of meta arrays and the proposed +``like=`` argument, it now becomes possible to handle cases involving creation +of non-NumPy arrays, which is likely the heaviest limitation Dask currently +faces from the ``__array_function__`` protocol. Backward Compatibility ---------------------- -- cgit v1.2.1 From 59723b6a76ed6b16b7fd72933b278aa25d09c08b Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Thu, 20 Aug 2020 10:29:32 +0100 Subject: BUG: Handle errors from the PyCapsure API This removes all uses of the `NpyCapsule` compatibility API. A lot of these call-sites would segfault if a bad capsule was introduced. These now raise `ValueError("... called with invalid PyCapsule object")`. --- numpy/core/src/multiarray/convert_datatype.c | 7 +++++-- numpy/core/src/multiarray/ctors.c | 14 ++++++++++---- numpy/core/src/multiarray/getset.c | 9 +++++++-- numpy/core/src/multiarray/multiarraymodule.c | 6 +++--- numpy/core/src/multiarray/scalartypes.c.src | 16 ++++++++++------ numpy/core/src/multiarray/usertypes.c | 2 +- numpy/core/src/umath/ufunc_object.c | 21 +++++++++++++++------ numpy/core/src/umath/ufunc_type_resolution.c | 27 +++++++++++++++------------ 8 files changed, 66 insertions(+), 36 deletions(-) diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 94cd1e5fa..5bbc5f08a 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -95,8 +95,11 @@ PyArray_GetCastFunc(PyArray_Descr *descr, int type_num) key = PyInt_FromLong(type_num); cobj = PyDict_GetItem(obj, key); Py_DECREF(key); - if (cobj && NpyCapsule_Check(cobj)) { - castfunc = NpyCapsule_AsVoidPtr(cobj); + if (cobj && PyCapsule_CheckExact(cobj)) { + castfunc = PyCapsule_GetPointer(cobj, NULL); + if (castfunc == NULL) { + return NULL; + } } } } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 15824e9e2..aa9b5f7cc 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -868,11 +868,14 @@ PyArray_NewFromDescr_int( func = PyObject_GetAttr((PyObject *)fa, npy_ma_str_array_finalize); if (func && func != Py_None) { - if (NpyCapsule_Check(func)) { + if (PyCapsule_CheckExact(func)) { /* A C-function is stored here */ PyArray_FinalizeFunc *cfunc; - cfunc = NpyCapsule_AsVoidPtr(func); + cfunc = PyCapsule_GetPointer(func, NULL); Py_DECREF(func); + if (cfunc == NULL) { + goto fail; + } if (cfunc((PyArrayObject *)fa, obj) < 0) { goto fail; } @@ -1747,7 +1750,7 @@ PyArray_FromStructInterface(PyObject *input) return Py_NotImplemented; } } - if (!NpyCapsule_Check(attr)) { + if (!PyCapsule_CheckExact(attr)) { if (PyType_Check(input) && PyObject_HasAttrString(attr, "__get__")) { /* * If the input is a class `attr` should be a property-like object. @@ -1759,7 +1762,10 @@ PyArray_FromStructInterface(PyObject *input) } goto fail; } - inter = NpyCapsule_AsVoidPtr(attr); + inter = PyCapsule_GetPointer(attr, NULL); + if (inter == NULL) { + goto fail; + } if (inter->two != 2) { goto fail; } diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 9066f52a8..f997c8436 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -621,7 +621,6 @@ static PyObject * array_struct_get(PyArrayObject *self) { PyArrayInterface *inter; - PyObject *ret; inter = (PyArrayInterface *)PyArray_malloc(sizeof(PyArrayInterface)); if (inter==NULL) { @@ -673,8 +672,14 @@ array_struct_get(PyArrayObject *self) else { inter->descr = NULL; } + PyObject *ret = PyCapsule_New(inter, NULL, gentype_struct_free); + if (ret == NULL) { + return NULL; + } Py_INCREF(self); - ret = NpyCapsule_FromVoidPtrAndDesc(inter, self, gentype_struct_free); + if (PyCapsule_SetContext(ret, self) < 0) { + return NULL; + } return ret; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 7c5ceb962..48f80403f 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4477,14 +4477,14 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { goto err; } - c_api = NpyCapsule_FromVoidPtr((void *)PyArray_API, NULL); + c_api = PyCapsule_New((void *)PyArray_API, NULL, NULL); if (c_api == NULL) { goto err; } PyDict_SetItemString(d, "_ARRAY_API", c_api); Py_DECREF(c_api); - c_api = NpyCapsule_FromVoidPtr((void *)PyUFunc_API, NULL); + c_api = PyCapsule_New((void *)PyUFunc_API, NULL, NULL); if (c_api == NULL) { goto err; } @@ -4540,7 +4540,7 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { } Py_DECREF(s); - s = NpyCapsule_FromVoidPtr((void *)_datetime_strings, NULL); + s = PyCapsule_New((void *)_datetime_strings, NULL, NULL); if (s == NULL) { goto err; } diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 58b9e2c30..4ba25b409 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -1147,12 +1147,16 @@ gentype_sizeof(PyObject *self) NPY_NO_EXPORT void gentype_struct_free(PyObject *ptr) { - PyArrayInterface *arrif; - PyObject *context; - - arrif = (PyArrayInterface*)PyCapsule_GetPointer(ptr, NULL); - context = (PyObject *)PyCapsule_GetContext(ptr); - Py_DECREF(context); + PyArrayInterface *arrif = (PyArrayInterface*)PyCapsule_GetPointer(ptr, NULL); + if (arrif == NULL) { + PyErr_WriteUnraisable(ptr); + return; + } + PyObject *context = (PyObject *)PyCapsule_GetContext(ptr); + if (context == NULL && PyErr_Occurred()) { + PyErr_WriteUnraisable(ptr); + } + Py_XDECREF(context); Py_XDECREF(arrif->descr); PyArray_free(arrif->shape); PyArray_free(arrif); diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 0c8d49970..f7bf5bf63 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -272,7 +272,7 @@ PyArray_RegisterCastFunc(PyArray_Descr *descr, int totype, if (PyErr_Occurred()) { return -1; } - cobj = NpyCapsule_FromVoidPtr((void *)castfunc, NULL); + cobj = PyCapsule_New((void *)castfunc, NULL, NULL); if (cobj == NULL) { Py_DECREF(key); return -1; diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index b47ccd291..8cfc6142e 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -3318,7 +3318,6 @@ get_binary_op_function(PyUFuncObject *ufunc, int *otype, void **out_innerloopdata) { int i; - PyUFunc_Loop1d *funcdata; NPY_UF_DBG_PRINT1("Getting binary op function for type number %d\n", *otype); @@ -3336,7 +3335,10 @@ get_binary_op_function(PyUFuncObject *ufunc, int *otype, return -1; } else if (obj != NULL) { - funcdata = (PyUFunc_Loop1d *)NpyCapsule_AsVoidPtr(obj); + PyUFunc_Loop1d *funcdata = PyCapsule_GetPointer(obj, NULL); + if (funcdata == NULL) { + return -1; + } while (funcdata != NULL) { int *types = funcdata->arg_types; @@ -5190,9 +5192,12 @@ PyUFunc_RegisterLoopForDescr(PyUFuncObject *ufunc, result = -1; } else { - PyUFunc_Loop1d *current; int cmp = 1; - current = (PyUFunc_Loop1d *)NpyCapsule_AsVoidPtr(cobj); + PyUFunc_Loop1d *current = PyCapsule_GetPointer(cobj, NULL); + if (current == NULL) { + result = -1; + goto done; + } while (current != NULL) { cmp = cmp_arg_types(current->arg_types, arg_typenums, ufunc->nargs); @@ -5226,6 +5231,7 @@ PyUFunc_RegisterLoopForDescr(PyUFuncObject *ufunc, } } +done: PyArray_free(arg_typenums); Py_DECREF(key); @@ -5294,7 +5300,7 @@ PyUFunc_RegisterLoopForType(PyUFuncObject *ufunc, } /* If it's not there, then make one and return. */ else if (cobj == NULL) { - cobj = NpyCapsule_FromVoidPtr((void *)funcdata, _loop1d_list_free); + cobj = PyCapsule_New((void *)funcdata, NULL, _loop1d_list_free); if (cobj == NULL) { goto fail; } @@ -5312,7 +5318,10 @@ PyUFunc_RegisterLoopForType(PyUFuncObject *ufunc, * is exactly like this one, then just replace. * Otherwise insert. */ - current = (PyUFunc_Loop1d *)NpyCapsule_AsVoidPtr(cobj); + current = PyCapsule_GetPointer(cobj, NULL); + if (current == NULL) { + goto fail; + } while (current != NULL) { cmp = cmp_arg_types(current->arg_types, newtypes, ufunc->nargs); if (cmp >= 0) { diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index ea20bb24f..0630b42c9 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -1336,7 +1336,6 @@ find_userloop(PyUFuncObject *ufunc, void **out_innerloopdata) { npy_intp i, nin = ufunc->nin, j, nargs = nin + ufunc->nout; - PyUFunc_Loop1d *funcdata; /* Use this to try to avoid repeating the same userdef loop search */ int last_userdef = -1; @@ -1368,9 +1367,11 @@ find_userloop(PyUFuncObject *ufunc, else if (obj == NULL) { continue; } - for (funcdata = (PyUFunc_Loop1d *)NpyCapsule_AsVoidPtr(obj); - funcdata != NULL; - funcdata = funcdata->next) { + PyUFunc_Loop1d *funcdata = PyCapsule_GetPointer(obj, NULL); + if (funcdata == NULL) { + return -1; + } + for (; funcdata != NULL; funcdata = funcdata->next) { int *types = funcdata->arg_types; for (j = 0; j < nargs; ++j) { @@ -1744,7 +1745,6 @@ linear_search_userloop_type_resolver(PyUFuncObject *self, char *out_err_dst_typecode) { npy_intp i, nop = self->nin + self->nout; - PyUFunc_Loop1d *funcdata; /* Use this to try to avoid repeating the same userdef loop search */ int last_userdef = -1; @@ -1776,9 +1776,11 @@ linear_search_userloop_type_resolver(PyUFuncObject *self, else if (obj == NULL) { continue; } - for (funcdata = (PyUFunc_Loop1d *)NpyCapsule_AsVoidPtr(obj); - funcdata != NULL; - funcdata = funcdata->next) { + PyUFunc_Loop1d *funcdata = PyCapsule_GetPointer(obj, NULL); + if (funcdata == NULL) { + return -1; + } + for (; funcdata != NULL; funcdata = funcdata->next) { int *types = funcdata->arg_types; switch (ufunc_loop_matches(self, op, input_casting, output_casting, @@ -1816,7 +1818,6 @@ type_tuple_userloop_type_resolver(PyUFuncObject *self, PyArray_Descr **out_dtype) { int i, j, nin = self->nin, nop = nin + self->nout; - PyUFunc_Loop1d *funcdata; /* Use this to try to avoid repeating the same userdef loop search */ int last_userdef = -1; @@ -1844,9 +1845,11 @@ type_tuple_userloop_type_resolver(PyUFuncObject *self, continue; } - for (funcdata = (PyUFunc_Loop1d *)NpyCapsule_AsVoidPtr(obj); - funcdata != NULL; - funcdata = funcdata->next) { + PyUFunc_Loop1d *funcdata = PyCapsule_GetPointer(obj, NULL); + if (funcdata == NULL) { + return -1; + } + for (; funcdata != NULL; funcdata = funcdata->next) { int *types = funcdata->arg_types; int matched = 1; -- cgit v1.2.1 From f7df11c52b48a7dd22cb4fc968d8af7cce4a2c5f Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 19 Aug 2020 22:11:23 +0100 Subject: MAINT: Split einsum into multiple files Putting the sumprod stuff in a separate file makes a handful of open PRs easier to review. The only changes to the large diffs here are: * Some defines and typedefs moved to headers * `NPY_VISIBILITY_HIDDEN` was added to the one function used from `sumprod` --- numpy/core/setup.py | 3 + numpy/core/src/multiarray/einsum.c.src | 1895 +---------------------- numpy/core/src/multiarray/einsum_debug.h | 28 + numpy/core/src/multiarray/einsum_sumprod.c.src | 1897 ++++++++++++++++++++++++ numpy/core/src/multiarray/einsum_sumprod.h | 12 + 5 files changed, 1942 insertions(+), 1893 deletions(-) create mode 100644 numpy/core/src/multiarray/einsum_debug.h create mode 100644 numpy/core/src/multiarray/einsum_sumprod.c.src create mode 100644 numpy/core/src/multiarray/einsum_sumprod.h diff --git a/numpy/core/setup.py b/numpy/core/setup.py index aede12080..a4a84397d 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -790,6 +790,8 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'descriptor.h'), join('src', 'multiarray', 'dtypemeta.h'), join('src', 'multiarray', 'dragon4.h'), + join('src', 'multiarray', 'einsum_debug.h'), + join('src', 'multiarray', 'einsum_sumprod.h'), join('src', 'multiarray', 'getset.h'), join('src', 'multiarray', 'hashdescr.h'), join('src', 'multiarray', 'iterators.h'), @@ -853,6 +855,7 @@ def configuration(parent_package='',top_path=None): join('src', 'multiarray', 'dragon4.c'), join('src', 'multiarray', 'dtype_transfer.c'), join('src', 'multiarray', 'einsum.c.src'), + join('src', 'multiarray', 'einsum_sumprod.c.src'), join('src', 'multiarray', 'flagsobject.c'), join('src', 'multiarray', 'getset.c'), join('src', 'multiarray', 'hashdescr.c'), diff --git a/numpy/core/src/multiarray/einsum.c.src b/numpy/core/src/multiarray/einsum.c.src index 2538e05c6..6ad375f67 100644 --- a/numpy/core/src/multiarray/einsum.c.src +++ b/numpy/core/src/multiarray/einsum.c.src @@ -16,7 +16,6 @@ #define _MULTIARRAYMODULE #include #include -#include #include #include @@ -25,1898 +24,8 @@ #include "common.h" #include "ctors.h" -#ifdef NPY_HAVE_SSE_INTRINSICS -#define EINSUM_USE_SSE1 1 -#else -#define EINSUM_USE_SSE1 0 -#endif - -#ifdef NPY_HAVE_SSE2_INTRINSICS -#define EINSUM_USE_SSE2 1 -#else -#define EINSUM_USE_SSE2 0 -#endif - -#if EINSUM_USE_SSE1 -#include -#endif - -#if EINSUM_USE_SSE2 -#include -#endif - -#define EINSUM_IS_SSE_ALIGNED(x) ((((npy_intp)x)&0xf) == 0) - -/********** PRINTF DEBUG TRACING **************/ -#define NPY_EINSUM_DBG_TRACING 0 - -#if NPY_EINSUM_DBG_TRACING -#define NPY_EINSUM_DBG_PRINT(s) printf("%s", s); -#define NPY_EINSUM_DBG_PRINT1(s, p1) printf(s, p1); -#define NPY_EINSUM_DBG_PRINT2(s, p1, p2) printf(s, p1, p2); -#define NPY_EINSUM_DBG_PRINT3(s, p1, p2, p3) printf(s); -#else -#define NPY_EINSUM_DBG_PRINT(s) -#define NPY_EINSUM_DBG_PRINT1(s, p1) -#define NPY_EINSUM_DBG_PRINT2(s, p1, p2) -#define NPY_EINSUM_DBG_PRINT3(s, p1, p2, p3) -#endif -/**********************************************/ - -/**begin repeat - * #name = byte, short, int, long, longlong, - * ubyte, ushort, uint, ulong, ulonglong, - * half, float, double, longdouble, - * cfloat, cdouble, clongdouble# - * #type = npy_byte, npy_short, npy_int, npy_long, npy_longlong, - * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, - * npy_half, npy_float, npy_double, npy_longdouble, - * npy_cfloat, npy_cdouble, npy_clongdouble# - * #temptype = npy_byte, npy_short, npy_int, npy_long, npy_longlong, - * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, - * npy_float, npy_float, npy_double, npy_longdouble, - * npy_float, npy_double, npy_longdouble# - * #to = ,,,,, - * ,,,,, - * npy_float_to_half,,,, - * ,,# - * #from = ,,,,, - * ,,,,, - * npy_half_to_float,,,, - * ,,# - * #complex = 0*5, - * 0*5, - * 0*4, - * 1*3# - * #float32 = 0*5, - * 0*5, - * 0,1,0,0, - * 0*3# - * #float64 = 0*5, - * 0*5, - * 0,0,1,0, - * 0*3# - */ - -/**begin repeat1 - * #nop = 1, 2, 3, 1000# - * #noplabel = one, two, three, any# - */ -static void -@name@_sum_of_products_@noplabel@(int nop, char **dataptr, - npy_intp const *strides, npy_intp count) -{ -#if (@nop@ == 1) || (@nop@ <= 3 && !@complex@) - char *data0 = dataptr[0]; - npy_intp stride0 = strides[0]; -#endif -#if (@nop@ == 2 || @nop@ == 3) && !@complex@ - char *data1 = dataptr[1]; - npy_intp stride1 = strides[1]; -#endif -#if (@nop@ == 3) && !@complex@ - char *data2 = dataptr[2]; - npy_intp stride2 = strides[2]; -#endif -#if (@nop@ == 1) || (@nop@ <= 3 && !@complex@) - char *data_out = dataptr[@nop@]; - npy_intp stride_out = strides[@nop@]; -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_@noplabel@ (%d)\n", (int)count); - - while (count--) { -#if !@complex@ -# if @nop@ == 1 - *(@type@ *)data_out = @to@(@from@(*(@type@ *)data0) + - @from@(*(@type@ *)data_out)); - data0 += stride0; - data_out += stride_out; -# elif @nop@ == 2 - *(@type@ *)data_out = @to@(@from@(*(@type@ *)data0) * - @from@(*(@type@ *)data1) + - @from@(*(@type@ *)data_out)); - data0 += stride0; - data1 += stride1; - data_out += stride_out; -# elif @nop@ == 3 - *(@type@ *)data_out = @to@(@from@(*(@type@ *)data0) * - @from@(*(@type@ *)data1) * - @from@(*(@type@ *)data2) + - @from@(*(@type@ *)data_out)); - data0 += stride0; - data1 += stride1; - data2 += stride2; - data_out += stride_out; -# else - @temptype@ temp = @from@(*(@type@ *)dataptr[0]); - int i; - for (i = 1; i < nop; ++i) { - temp *= @from@(*(@type@ *)dataptr[i]); - } - *(@type@ *)dataptr[nop] = @to@(temp + - @from@(*(@type@ *)dataptr[i])); - for (i = 0; i <= nop; ++i) { - dataptr[i] += strides[i]; - } -# endif -#else /* complex */ -# if @nop@ == 1 - ((@temptype@ *)data_out)[0] = ((@temptype@ *)data0)[0] + - ((@temptype@ *)data_out)[0]; - ((@temptype@ *)data_out)[1] = ((@temptype@ *)data0)[1] + - ((@temptype@ *)data_out)[1]; - data0 += stride0; - data_out += stride_out; -# else -# if @nop@ <= 3 -#define _SUMPROD_NOP @nop@ -# else -#define _SUMPROD_NOP nop -# endif - @temptype@ re, im, tmp; - int i; - re = ((@temptype@ *)dataptr[0])[0]; - im = ((@temptype@ *)dataptr[0])[1]; - for (i = 1; i < _SUMPROD_NOP; ++i) { - tmp = re * ((@temptype@ *)dataptr[i])[0] - - im * ((@temptype@ *)dataptr[i])[1]; - im = re * ((@temptype@ *)dataptr[i])[1] + - im * ((@temptype@ *)dataptr[i])[0]; - re = tmp; - } - ((@temptype@ *)dataptr[_SUMPROD_NOP])[0] = re + - ((@temptype@ *)dataptr[_SUMPROD_NOP])[0]; - ((@temptype@ *)dataptr[_SUMPROD_NOP])[1] = im + - ((@temptype@ *)dataptr[_SUMPROD_NOP])[1]; - - for (i = 0; i <= _SUMPROD_NOP; ++i) { - dataptr[i] += strides[i]; - } -#undef _SUMPROD_NOP -# endif -#endif - } -} - -#if @nop@ == 1 - -static void -@name@_sum_of_products_contig_one(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @type@ *data0 = (@type@ *)dataptr[0]; - @type@ *data_out = (@type@ *)dataptr[1]; - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_one (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: -#if !@complex@ - data_out[@i@] = @to@(@from@(data0[@i@]) + - @from@(data_out[@i@])); -#else - ((@temptype@ *)data_out + 2*@i@)[0] = - ((@temptype@ *)data0 + 2*@i@)[0] + - ((@temptype@ *)data_out + 2*@i@)[0]; - ((@temptype@ *)data_out + 2*@i@)[1] = - ((@temptype@ *)data0 + 2*@i@)[1] + - ((@temptype@ *)data_out + 2*@i@)[1]; -#endif -/**end repeat2**/ - case 0: - return; - } - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ -#if !@complex@ - data_out[@i@] = @to@(@from@(data0[@i@]) + - @from@(data_out[@i@])); -#else /* complex */ - ((@temptype@ *)data_out + 2*@i@)[0] = - ((@temptype@ *)data0 + 2*@i@)[0] + - ((@temptype@ *)data_out + 2*@i@)[0]; - ((@temptype@ *)data_out + 2*@i@)[1] = - ((@temptype@ *)data0 + 2*@i@)[1] + - ((@temptype@ *)data_out + 2*@i@)[1]; -#endif -/**end repeat2**/ - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -#elif @nop@ == 2 && !@complex@ - -static void -@name@_sum_of_products_contig_two(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @type@ *data0 = (@type@ *)dataptr[0]; - @type@ *data1 = (@type@ *)dataptr[1]; - @type@ *data_out = (@type@ *)dataptr[2]; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, b; -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, b; -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - data_out[@i@] = @to@(@from@(data0[@i@]) * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ - case 0: - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1) && - EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_load_ps(data0+@i@), _mm_load_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); - _mm_store_ps(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1) && - EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_load_pd(data0+@i@), _mm_load_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); - _mm_store_pd(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), _mm_loadu_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); - _mm_storeu_ps(data_out+@i@, b); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), _mm_loadu_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); - _mm_storeu_pd(data_out+@i@, b); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(@from@(data0[@i@]) * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ -#endif - data0 += 8; - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -/* Some extra specializations for the two operand case */ -static void -@name@_sum_of_products_stride0_contig_outcontig_two(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @temptype@ value0 = @from@(*(@type@ *)dataptr[0]); - @type@ *data1 = (@type@ *)dataptr[1]; - @type@ *data_out = (@type@ *)dataptr[2]; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, b, value0_sse; -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, b, value0_sse; -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_stride0_contig_outcontig_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - data_out[@i@] = @to@(value0 * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ - case 0: - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - value0_sse = _mm_set_ps1(value0); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(value0_sse, _mm_load_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); - _mm_store_ps(data_out+@i@, b); -/**end repeat2**/ - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - if (count > 0) { - goto finish_after_unrolled_loop; - } - else { - return; - } - } -#elif EINSUM_USE_SSE2 && @float64@ - value0_sse = _mm_set1_pd(value0); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(value0_sse, _mm_load_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); - _mm_store_pd(data_out+@i@, b); -/**end repeat2**/ - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - if (count > 0) { - goto finish_after_unrolled_loop; - } - else { - return; - } - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(value0_sse, _mm_loadu_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); - _mm_storeu_ps(data_out+@i@, b); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(value0_sse, _mm_loadu_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); - _mm_storeu_pd(data_out+@i@, b); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(value0 * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ -#endif - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - if (count > 0) { - goto finish_after_unrolled_loop; - } -} - -static void -@name@_sum_of_products_contig_stride0_outcontig_two(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @type@ *data0 = (@type@ *)dataptr[0]; - @temptype@ value1 = @from@(*(@type@ *)dataptr[1]); - @type@ *data_out = (@type@ *)dataptr[2]; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, b, value1_sse; -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, b, value1_sse; -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_stride0_outcontig_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - data_out[@i@] = @to@(@from@(data0[@i@])* - value1 + - @from@(data_out[@i@])); -/**end repeat2**/ - case 0: - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - value1_sse = _mm_set_ps1(value1); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_load_ps(data0+@i@), value1_sse); - b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); - _mm_store_ps(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - value1_sse = _mm_set1_pd(value1); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_load_pd(data0+@i@), value1_sse); - b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); - _mm_store_pd(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), value1_sse); - b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); - _mm_storeu_ps(data_out+@i@, b); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), value1_sse); - b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); - _mm_storeu_pd(data_out+@i@, b); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(@from@(data0[@i@])* - value1 + - @from@(data_out[@i@])); -/**end repeat2**/ -#endif - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -static void -@name@_sum_of_products_contig_contig_outstride0_two(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @type@ *data0 = (@type@ *)dataptr[0]; - @type@ *data1 = (@type@ *)dataptr[1]; - @temptype@ accum = 0; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_contig_outstride0_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - accum += @from@(data0[@i@]) * @from@(data1[@i@]); -/**end repeat2**/ - case 0: - *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum); - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_ps(_mm_load_ps(data0+@i@), _mm_load_ps(data1+@i@)); - accum_sse = _mm_add_ps(accum_sse, a); -/**end repeat2**/ - data0 += 8; - data1 += 8; - } - - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_pd(_mm_load_pd(data0+@i@), _mm_load_pd(data1+@i@)); - accum_sse = _mm_add_pd(accum_sse, a); -/**end repeat2**/ - data0 += 8; - data1 += 8; - } - - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), _mm_loadu_ps(data1+@i@)); - accum_sse = _mm_add_ps(accum_sse, a); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), _mm_loadu_pd(data1+@i@)); - accum_sse = _mm_add_pd(accum_sse, a); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - accum += @from@(data0[@i@]) * @from@(data1[@i@]); -/**end repeat2**/ -#endif - data0 += 8; - data1 += 8; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -static void -@name@_sum_of_products_stride0_contig_outstride0_two(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @temptype@ value0 = @from@(*(@type@ *)dataptr[0]); - @type@ *data1 = (@type@ *)dataptr[1]; - @temptype@ accum = 0; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_stride0_contig_outstride0_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - accum += @from@(data1[@i@]); -/**end repeat2**/ - case 0: - *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + value0 * accum); - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data1+@i@)); -/**end repeat2**/ - data1 += 8; - } - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data1+@i@)); -/**end repeat2**/ - data1 += 8; - } - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data1+@i@)); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data1+@i@)); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - accum += @from@(data1[@i@]); -/**end repeat2**/ -#endif - data1 += 8; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -static void -@name@_sum_of_products_contig_stride0_outstride0_two(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @type@ *data0 = (@type@ *)dataptr[0]; - @temptype@ value1 = @from@(*(@type@ *)dataptr[1]); - @temptype@ accum = 0; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_stride0_outstride0_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - accum += @from@(data0[@i@]); -/**end repeat2**/ - case 0: - *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum * value1); - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data0+@i@)); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data0+@i@)); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - accum += @from@(data0[@i@]); -/**end repeat2**/ -#endif - data0 += 8; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -#elif @nop@ == 3 && !@complex@ - -static void -@name@_sum_of_products_contig_three(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - @type@ *data0 = (@type@ *)dataptr[0]; - @type@ *data1 = (@type@ *)dataptr[1]; - @type@ *data2 = (@type@ *)dataptr[2]; - @type@ *data_out = (@type@ *)dataptr[3]; - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(@from@(data0[@i@]) * - @from@(data1[@i@]) * - @from@(data2[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ - data0 += 8; - data1 += 8; - data2 += 8; - data_out += 8; - } - - /* Finish off the loop */ - -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - if (count-- == 0) { - return; - } - data_out[@i@] = @to@(@from@(data0[@i@]) * - @from@(data1[@i@]) * - @from@(data2[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ -} - -#else /* @nop@ > 3 || @complex */ - -static void -@name@_sum_of_products_contig_@noplabel@(int nop, char **dataptr, - npy_intp const *NPY_UNUSED(strides), npy_intp count) -{ - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_@noplabel@ (%d)\n", - (int)count); - - while (count--) { -#if !@complex@ - @temptype@ temp = @from@(*(@type@ *)dataptr[0]); - int i; - for (i = 1; i < nop; ++i) { - temp *= @from@(*(@type@ *)dataptr[i]); - } - *(@type@ *)dataptr[nop] = @to@(temp + - @from@(*(@type@ *)dataptr[i])); - for (i = 0; i <= nop; ++i) { - dataptr[i] += sizeof(@type@); - } -#else /* complex */ -# if @nop@ <= 3 -# define _SUMPROD_NOP @nop@ -# else -# define _SUMPROD_NOP nop -# endif - @temptype@ re, im, tmp; - int i; - re = ((@temptype@ *)dataptr[0])[0]; - im = ((@temptype@ *)dataptr[0])[1]; - for (i = 1; i < _SUMPROD_NOP; ++i) { - tmp = re * ((@temptype@ *)dataptr[i])[0] - - im * ((@temptype@ *)dataptr[i])[1]; - im = re * ((@temptype@ *)dataptr[i])[1] + - im * ((@temptype@ *)dataptr[i])[0]; - re = tmp; - } - ((@temptype@ *)dataptr[_SUMPROD_NOP])[0] = re + - ((@temptype@ *)dataptr[_SUMPROD_NOP])[0]; - ((@temptype@ *)dataptr[_SUMPROD_NOP])[1] = im + - ((@temptype@ *)dataptr[_SUMPROD_NOP])[1]; - - for (i = 0; i <= _SUMPROD_NOP; ++i) { - dataptr[i] += sizeof(@type@); - } -# undef _SUMPROD_NOP -#endif - } -} - -#endif /* functions for various @nop@ */ - -#if @nop@ == 1 - -static void -@name@_sum_of_products_contig_outstride0_one(int nop, char **dataptr, - npy_intp const *strides, npy_intp count) -{ -#if @complex@ - @temptype@ accum_re = 0, accum_im = 0; - @temptype@ *data0 = (@temptype@ *)dataptr[0]; -#else - @temptype@ accum = 0; - @type@ *data0 = (@type@ *)dataptr[0]; -#endif - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_outstride0_one (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: -#if !@complex@ - accum += @from@(data0[@i@]); -#else /* complex */ - accum_re += data0[2*@i@+0]; - accum_im += data0[2*@i@+1]; -#endif -/**end repeat2**/ - case 0: -#if @complex@ - ((@temptype@ *)dataptr[1])[0] += accum_re; - ((@temptype@ *)dataptr[1])[1] += accum_im; -#else - *((@type@ *)dataptr[1]) = @to@(accum + - @from@(*((@type@ *)dataptr[1]))); -#endif - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data0+@i@)); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data0+@i@)); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ -# if !@complex@ - accum += @from@(data0[@i@]); -# else /* complex */ - accum_re += data0[2*@i@+0]; - accum_im += data0[2*@i@+1]; -# endif -/**end repeat2**/ -#endif - -#if !@complex@ - data0 += 8; -#else - data0 += 8*2; -#endif - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; -} - -#endif /* @nop@ == 1 */ - -static void -@name@_sum_of_products_outstride0_@noplabel@(int nop, char **dataptr, - npy_intp const *strides, npy_intp count) -{ -#if @complex@ - @temptype@ accum_re = 0, accum_im = 0; -#else - @temptype@ accum = 0; -#endif - -#if (@nop@ == 1) || (@nop@ <= 3 && !@complex@) - char *data0 = dataptr[0]; - npy_intp stride0 = strides[0]; -#endif -#if (@nop@ == 2 || @nop@ == 3) && !@complex@ - char *data1 = dataptr[1]; - npy_intp stride1 = strides[1]; -#endif -#if (@nop@ == 3) && !@complex@ - char *data2 = dataptr[2]; - npy_intp stride2 = strides[2]; -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_outstride0_@noplabel@ (%d)\n", - (int)count); - - while (count--) { -#if !@complex@ -# if @nop@ == 1 - accum += @from@(*(@type@ *)data0); - data0 += stride0; -# elif @nop@ == 2 - accum += @from@(*(@type@ *)data0) * - @from@(*(@type@ *)data1); - data0 += stride0; - data1 += stride1; -# elif @nop@ == 3 - accum += @from@(*(@type@ *)data0) * - @from@(*(@type@ *)data1) * - @from@(*(@type@ *)data2); - data0 += stride0; - data1 += stride1; - data2 += stride2; -# else - @temptype@ temp = @from@(*(@type@ *)dataptr[0]); - int i; - for (i = 1; i < nop; ++i) { - temp *= @from@(*(@type@ *)dataptr[i]); - } - accum += temp; - for (i = 0; i < nop; ++i) { - dataptr[i] += strides[i]; - } -# endif -#else /* complex */ -# if @nop@ == 1 - accum_re += ((@temptype@ *)data0)[0]; - accum_im += ((@temptype@ *)data0)[1]; - data0 += stride0; -# else -# if @nop@ <= 3 -#define _SUMPROD_NOP @nop@ -# else -#define _SUMPROD_NOP nop -# endif - @temptype@ re, im, tmp; - int i; - re = ((@temptype@ *)dataptr[0])[0]; - im = ((@temptype@ *)dataptr[0])[1]; - for (i = 1; i < _SUMPROD_NOP; ++i) { - tmp = re * ((@temptype@ *)dataptr[i])[0] - - im * ((@temptype@ *)dataptr[i])[1]; - im = re * ((@temptype@ *)dataptr[i])[1] + - im * ((@temptype@ *)dataptr[i])[0]; - re = tmp; - } - accum_re += re; - accum_im += im; - for (i = 0; i < _SUMPROD_NOP; ++i) { - dataptr[i] += strides[i]; - } -#undef _SUMPROD_NOP -# endif -#endif - } - -#if @complex@ -# if @nop@ <= 3 - ((@temptype@ *)dataptr[@nop@])[0] += accum_re; - ((@temptype@ *)dataptr[@nop@])[1] += accum_im; -# else - ((@temptype@ *)dataptr[nop])[0] += accum_re; - ((@temptype@ *)dataptr[nop])[1] += accum_im; -# endif -#else -# if @nop@ <= 3 - *((@type@ *)dataptr[@nop@]) = @to@(accum + - @from@(*((@type@ *)dataptr[@nop@]))); -# else - *((@type@ *)dataptr[nop]) = @to@(accum + - @from@(*((@type@ *)dataptr[nop]))); -# endif -#endif - -} - -/**end repeat1**/ - -/**end repeat**/ - - -/* Do OR of ANDs for the boolean type */ - -/**begin repeat - * #nop = 1, 2, 3, 1000# - * #noplabel = one, two, three, any# - */ - -static void -bool_sum_of_products_@noplabel@(int nop, char **dataptr, - npy_intp const *strides, npy_intp count) -{ -#if (@nop@ <= 3) - char *data0 = dataptr[0]; - npy_intp stride0 = strides[0]; -#endif -#if (@nop@ == 2 || @nop@ == 3) - char *data1 = dataptr[1]; - npy_intp stride1 = strides[1]; -#endif -#if (@nop@ == 3) - char *data2 = dataptr[2]; - npy_intp stride2 = strides[2]; -#endif -#if (@nop@ <= 3) - char *data_out = dataptr[@nop@]; - npy_intp stride_out = strides[@nop@]; -#endif - - while (count--) { -#if @nop@ == 1 - *(npy_bool *)data_out = *(npy_bool *)data0 || - *(npy_bool *)data_out; - data0 += stride0; - data_out += stride_out; -#elif @nop@ == 2 - *(npy_bool *)data_out = (*(npy_bool *)data0 && - *(npy_bool *)data1) || - *(npy_bool *)data_out; - data0 += stride0; - data1 += stride1; - data_out += stride_out; -#elif @nop@ == 3 - *(npy_bool *)data_out = (*(npy_bool *)data0 && - *(npy_bool *)data1 && - *(npy_bool *)data2) || - *(npy_bool *)data_out; - data0 += stride0; - data1 += stride1; - data2 += stride2; - data_out += stride_out; -#else - npy_bool temp = *(npy_bool *)dataptr[0]; - int i; - for (i = 1; i < nop; ++i) { - temp = temp && *(npy_bool *)dataptr[i]; - } - *(npy_bool *)dataptr[nop] = temp || *(npy_bool *)dataptr[i]; - for (i = 0; i <= nop; ++i) { - dataptr[i] += strides[i]; - } -#endif - } -} - -static void -bool_sum_of_products_contig_@noplabel@(int nop, char **dataptr, - npy_intp const *strides, npy_intp count) -{ -#if (@nop@ <= 3) - char *data0 = dataptr[0]; -#endif -#if (@nop@ == 2 || @nop@ == 3) - char *data1 = dataptr[1]; -#endif -#if (@nop@ == 3) - char *data2 = dataptr[2]; -#endif -#if (@nop@ <= 3) - char *data_out = dataptr[@nop@]; -#endif - -#if (@nop@ <= 3) -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat1 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: -# if @nop@ == 1 - ((npy_bool *)data_out)[@i@] = ((npy_bool *)data0)[@i@] || - ((npy_bool *)data_out)[@i@]; -# elif @nop@ == 2 - ((npy_bool *)data_out)[@i@] = - (((npy_bool *)data0)[@i@] && - ((npy_bool *)data1)[@i@]) || - ((npy_bool *)data_out)[@i@]; -# elif @nop@ == 3 - ((npy_bool *)data_out)[@i@] = - (((npy_bool *)data0)[@i@] && - ((npy_bool *)data1)[@i@] && - ((npy_bool *)data2)[@i@]) || - ((npy_bool *)data_out)[@i@]; -# endif -/**end repeat1**/ - case 0: - return; - } -#endif - -/* Unroll the loop by 8 for fixed-size nop */ -#if (@nop@ <= 3) - while (count >= 8) { - count -= 8; -#else - while (count--) { -#endif - -# if @nop@ == 1 -/**begin repeat1 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - *((npy_bool *)data_out + @i@) = (*((npy_bool *)data0 + @i@)) || - (*((npy_bool *)data_out + @i@)); -/**end repeat1**/ - data0 += 8*sizeof(npy_bool); - data_out += 8*sizeof(npy_bool); -# elif @nop@ == 2 -/**begin repeat1 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - *((npy_bool *)data_out + @i@) = - ((*((npy_bool *)data0 + @i@)) && - (*((npy_bool *)data1 + @i@))) || - (*((npy_bool *)data_out + @i@)); -/**end repeat1**/ - data0 += 8*sizeof(npy_bool); - data1 += 8*sizeof(npy_bool); - data_out += 8*sizeof(npy_bool); -# elif @nop@ == 3 -/**begin repeat1 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - *((npy_bool *)data_out + @i@) = - ((*((npy_bool *)data0 + @i@)) && - (*((npy_bool *)data1 + @i@)) && - (*((npy_bool *)data2 + @i@))) || - (*((npy_bool *)data_out + @i@)); -/**end repeat1**/ - data0 += 8*sizeof(npy_bool); - data1 += 8*sizeof(npy_bool); - data2 += 8*sizeof(npy_bool); - data_out += 8*sizeof(npy_bool); -# else - npy_bool temp = *(npy_bool *)dataptr[0]; - int i; - for (i = 1; i < nop; ++i) { - temp = temp && *(npy_bool *)dataptr[i]; - } - *(npy_bool *)dataptr[nop] = temp || *(npy_bool *)dataptr[i]; - for (i = 0; i <= nop; ++i) { - dataptr[i] += sizeof(npy_bool); - } -# endif - } - - /* If the loop was unrolled, we need to finish it off */ -#if (@nop@ <= 3) - goto finish_after_unrolled_loop; -#endif -} - -static void -bool_sum_of_products_outstride0_@noplabel@(int nop, char **dataptr, - npy_intp const *strides, npy_intp count) -{ - npy_bool accum = 0; - -#if (@nop@ <= 3) - char *data0 = dataptr[0]; - npy_intp stride0 = strides[0]; -#endif -#if (@nop@ == 2 || @nop@ == 3) - char *data1 = dataptr[1]; - npy_intp stride1 = strides[1]; -#endif -#if (@nop@ == 3) - char *data2 = dataptr[2]; - npy_intp stride2 = strides[2]; -#endif - - while (count--) { -#if @nop@ == 1 - accum = *(npy_bool *)data0 || accum; - data0 += stride0; -#elif @nop@ == 2 - accum = (*(npy_bool *)data0 && *(npy_bool *)data1) || accum; - data0 += stride0; - data1 += stride1; -#elif @nop@ == 3 - accum = (*(npy_bool *)data0 && - *(npy_bool *)data1 && - *(npy_bool *)data2) || accum; - data0 += stride0; - data1 += stride1; - data2 += stride2; -#else - npy_bool temp = *(npy_bool *)dataptr[0]; - int i; - for (i = 1; i < nop; ++i) { - temp = temp && *(npy_bool *)dataptr[i]; - } - accum = temp || accum; - for (i = 0; i <= nop; ++i) { - dataptr[i] += strides[i]; - } -#endif - } - -# if @nop@ <= 3 - *((npy_bool *)dataptr[@nop@]) = accum || *((npy_bool *)dataptr[@nop@]); -# else - *((npy_bool *)dataptr[nop]) = accum || *((npy_bool *)dataptr[nop]); -# endif -} - -/**end repeat**/ - -typedef void (*sum_of_products_fn)(int, char **, npy_intp const*, npy_intp); - -/* These tables need to match up with the type enum */ -static sum_of_products_fn -_contig_outstride0_unary_specialization_table[NPY_NTYPES] = { -/**begin repeat - * #name = bool, - * byte, ubyte, - * short, ushort, - * int, uint, - * long, ulong, - * longlong, ulonglong, - * float, double, longdouble, - * cfloat, cdouble, clongdouble, - * object, string, unicode, void, - * datetime, timedelta, half# - * #use = 0, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, 1, - * 1, 1, 1, - * 0, 0, 0, 0, - * 0, 0, 1# - */ -#if @use@ - &@name@_sum_of_products_contig_outstride0_one, -#else - NULL, -#endif -/**end repeat**/ -}; /* End of _contig_outstride0_unary_specialization_table */ - -static sum_of_products_fn _binary_specialization_table[NPY_NTYPES][5] = { -/**begin repeat - * #name = bool, - * byte, ubyte, - * short, ushort, - * int, uint, - * long, ulong, - * longlong, ulonglong, - * float, double, longdouble, - * cfloat, cdouble, clongdouble, - * object, string, unicode, void, - * datetime, timedelta, half# - * #use = 0, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, 1, - * 0, 0, 0, - * 0, 0, 0, 0, - * 0, 0, 1# - */ -#if @use@ -{ - &@name@_sum_of_products_stride0_contig_outstride0_two, - &@name@_sum_of_products_stride0_contig_outcontig_two, - &@name@_sum_of_products_contig_stride0_outstride0_two, - &@name@_sum_of_products_contig_stride0_outcontig_two, - &@name@_sum_of_products_contig_contig_outstride0_two, -}, -#else - {NULL, NULL, NULL, NULL, NULL}, -#endif -/**end repeat**/ -}; /* End of _binary_specialization_table */ - -static sum_of_products_fn _outstride0_specialized_table[NPY_NTYPES][4] = { -/**begin repeat - * #name = bool, - * byte, ubyte, - * short, ushort, - * int, uint, - * long, ulong, - * longlong, ulonglong, - * float, double, longdouble, - * cfloat, cdouble, clongdouble, - * object, string, unicode, void, - * datetime, timedelta, half# - * #use = 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, 1, - * 1, 1, 1, - * 0, 0, 0, 0, - * 0, 0, 1# - */ -#if @use@ -{ - &@name@_sum_of_products_outstride0_any, - &@name@_sum_of_products_outstride0_one, - &@name@_sum_of_products_outstride0_two, - &@name@_sum_of_products_outstride0_three -}, -#else - {NULL, NULL, NULL, NULL}, -#endif -/**end repeat**/ -}; /* End of _outstride0_specialized_table */ - -static sum_of_products_fn _allcontig_specialized_table[NPY_NTYPES][4] = { -/**begin repeat - * #name = bool, - * byte, ubyte, - * short, ushort, - * int, uint, - * long, ulong, - * longlong, ulonglong, - * float, double, longdouble, - * cfloat, cdouble, clongdouble, - * object, string, unicode, void, - * datetime, timedelta, half# - * #use = 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, 1, - * 1, 1, 1, - * 0, 0, 0, 0, - * 0, 0, 1# - */ -#if @use@ -{ - &@name@_sum_of_products_contig_any, - &@name@_sum_of_products_contig_one, - &@name@_sum_of_products_contig_two, - &@name@_sum_of_products_contig_three -}, -#else - {NULL, NULL, NULL, NULL}, -#endif -/**end repeat**/ -}; /* End of _allcontig_specialized_table */ - -static sum_of_products_fn _unspecialized_table[NPY_NTYPES][4] = { -/**begin repeat - * #name = bool, - * byte, ubyte, - * short, ushort, - * int, uint, - * long, ulong, - * longlong, ulonglong, - * float, double, longdouble, - * cfloat, cdouble, clongdouble, - * object, string, unicode, void, - * datetime, timedelta, half# - * #use = 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, - * 1, 1, 1, - * 1, 1, 1, - * 0, 0, 0, 0, - * 0, 0, 1# - */ -#if @use@ -{ - &@name@_sum_of_products_any, - &@name@_sum_of_products_one, - &@name@_sum_of_products_two, - &@name@_sum_of_products_three -}, -#else - {NULL, NULL, NULL, NULL}, -#endif -/**end repeat**/ -}; /* End of _unnspecialized_table */ - -static sum_of_products_fn -get_sum_of_products_function(int nop, int type_num, - npy_intp itemsize, npy_intp const *fixed_strides) -{ - int iop; - - if (type_num >= NPY_NTYPES) { - return NULL; - } - - /* contiguous reduction */ - if (nop == 1 && fixed_strides[0] == itemsize && fixed_strides[1] == 0) { - sum_of_products_fn ret = - _contig_outstride0_unary_specialization_table[type_num]; - if (ret != NULL) { - return ret; - } - } - - /* nop of 2 has more specializations */ - if (nop == 2) { - /* Encode the zero/contiguous strides */ - int code; - code = (fixed_strides[0] == 0) ? 0 : - (fixed_strides[0] == itemsize) ? 2*2*1 : 8; - code += (fixed_strides[1] == 0) ? 0 : - (fixed_strides[1] == itemsize) ? 2*1 : 8; - code += (fixed_strides[2] == 0) ? 0 : - (fixed_strides[2] == itemsize) ? 1 : 8; - if (code >= 2 && code < 7) { - sum_of_products_fn ret = - _binary_specialization_table[type_num][code-2]; - if (ret != NULL) { - return ret; - } - } - } - - /* Inner loop with an output stride of 0 */ - if (fixed_strides[nop] == 0) { - return _outstride0_specialized_table[type_num][nop <= 3 ? nop : 0]; - } - - /* Check for all contiguous */ - for (iop = 0; iop < nop + 1; ++iop) { - if (fixed_strides[iop] != itemsize) { - break; - } - } - - /* Contiguous loop */ - if (iop == nop + 1) { - return _allcontig_specialized_table[type_num][nop <= 3 ? nop : 0]; - } - - /* None of the above specializations caught it, general loops */ - return _unspecialized_table[type_num][nop <= 3 ? nop : 0]; -} +#include "einsum_sumprod.h" +#include "einsum_debug.h" /* diff --git a/numpy/core/src/multiarray/einsum_debug.h b/numpy/core/src/multiarray/einsum_debug.h new file mode 100644 index 000000000..9aa81fcbd --- /dev/null +++ b/numpy/core/src/multiarray/einsum_debug.h @@ -0,0 +1,28 @@ +/* + * This file provides debug macros used by the other einsum files. + * + * Copyright (c) 2011 by Mark Wiebe (mwwiebe@gmail.com) + * The University of British Columbia + * + * See LICENSE.txt for the license. + */ +#ifndef _NPY_MULTIARRAY_EINSUM_DEBUG_H +#define _NPY_MULTIARRAY_EINSUM_DEBUG_H + +/********** PRINTF DEBUG TRACING **************/ +#define NPY_EINSUM_DBG_TRACING 0 + +#if NPY_EINSUM_DBG_TRACING +#include +#define NPY_EINSUM_DBG_PRINT(s) printf("%s", s); +#define NPY_EINSUM_DBG_PRINT1(s, p1) printf(s, p1); +#define NPY_EINSUM_DBG_PRINT2(s, p1, p2) printf(s, p1, p2); +#define NPY_EINSUM_DBG_PRINT3(s, p1, p2, p3) printf(s); +#else +#define NPY_EINSUM_DBG_PRINT(s) +#define NPY_EINSUM_DBG_PRINT1(s, p1) +#define NPY_EINSUM_DBG_PRINT2(s, p1, p2) +#define NPY_EINSUM_DBG_PRINT3(s, p1, p2, p3) +#endif + +#endif diff --git a/numpy/core/src/multiarray/einsum_sumprod.c.src b/numpy/core/src/multiarray/einsum_sumprod.c.src new file mode 100644 index 000000000..c58e74287 --- /dev/null +++ b/numpy/core/src/multiarray/einsum_sumprod.c.src @@ -0,0 +1,1897 @@ +/* + * This file provides optimized sum of product implementations used internally + * by einsum. + * + * Copyright (c) 2011 by Mark Wiebe (mwwiebe@gmail.com) + * The University of British Columbia + * + * See LICENSE.txt for the license. + */ + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include +#include /* for NPY_NTYPES */ +#include + +#include "einsum_sumprod.h" +#include "einsum_debug.h" + + +#ifdef NPY_HAVE_SSE_INTRINSICS +#define EINSUM_USE_SSE1 1 +#else +#define EINSUM_USE_SSE1 0 +#endif + +#ifdef NPY_HAVE_SSE2_INTRINSICS +#define EINSUM_USE_SSE2 1 +#else +#define EINSUM_USE_SSE2 0 +#endif + +#if EINSUM_USE_SSE1 +#include +#endif + +#if EINSUM_USE_SSE2 +#include +#endif + +#define EINSUM_IS_SSE_ALIGNED(x) ((((npy_intp)x)&0xf) == 0) + +/**********************************************/ + +/**begin repeat + * #name = byte, short, int, long, longlong, + * ubyte, ushort, uint, ulong, ulonglong, + * half, float, double, longdouble, + * cfloat, cdouble, clongdouble# + * #type = npy_byte, npy_short, npy_int, npy_long, npy_longlong, + * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, + * npy_half, npy_float, npy_double, npy_longdouble, + * npy_cfloat, npy_cdouble, npy_clongdouble# + * #temptype = npy_byte, npy_short, npy_int, npy_long, npy_longlong, + * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, + * npy_float, npy_float, npy_double, npy_longdouble, + * npy_float, npy_double, npy_longdouble# + * #to = ,,,,, + * ,,,,, + * npy_float_to_half,,,, + * ,,# + * #from = ,,,,, + * ,,,,, + * npy_half_to_float,,,, + * ,,# + * #complex = 0*5, + * 0*5, + * 0*4, + * 1*3# + * #float32 = 0*5, + * 0*5, + * 0,1,0,0, + * 0*3# + * #float64 = 0*5, + * 0*5, + * 0,0,1,0, + * 0*3# + */ + +/**begin repeat1 + * #nop = 1, 2, 3, 1000# + * #noplabel = one, two, three, any# + */ +static void +@name@_sum_of_products_@noplabel@(int nop, char **dataptr, + npy_intp const *strides, npy_intp count) +{ +#if (@nop@ == 1) || (@nop@ <= 3 && !@complex@) + char *data0 = dataptr[0]; + npy_intp stride0 = strides[0]; +#endif +#if (@nop@ == 2 || @nop@ == 3) && !@complex@ + char *data1 = dataptr[1]; + npy_intp stride1 = strides[1]; +#endif +#if (@nop@ == 3) && !@complex@ + char *data2 = dataptr[2]; + npy_intp stride2 = strides[2]; +#endif +#if (@nop@ == 1) || (@nop@ <= 3 && !@complex@) + char *data_out = dataptr[@nop@]; + npy_intp stride_out = strides[@nop@]; +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_@noplabel@ (%d)\n", (int)count); + + while (count--) { +#if !@complex@ +# if @nop@ == 1 + *(@type@ *)data_out = @to@(@from@(*(@type@ *)data0) + + @from@(*(@type@ *)data_out)); + data0 += stride0; + data_out += stride_out; +# elif @nop@ == 2 + *(@type@ *)data_out = @to@(@from@(*(@type@ *)data0) * + @from@(*(@type@ *)data1) + + @from@(*(@type@ *)data_out)); + data0 += stride0; + data1 += stride1; + data_out += stride_out; +# elif @nop@ == 3 + *(@type@ *)data_out = @to@(@from@(*(@type@ *)data0) * + @from@(*(@type@ *)data1) * + @from@(*(@type@ *)data2) + + @from@(*(@type@ *)data_out)); + data0 += stride0; + data1 += stride1; + data2 += stride2; + data_out += stride_out; +# else + @temptype@ temp = @from@(*(@type@ *)dataptr[0]); + int i; + for (i = 1; i < nop; ++i) { + temp *= @from@(*(@type@ *)dataptr[i]); + } + *(@type@ *)dataptr[nop] = @to@(temp + + @from@(*(@type@ *)dataptr[i])); + for (i = 0; i <= nop; ++i) { + dataptr[i] += strides[i]; + } +# endif +#else /* complex */ +# if @nop@ == 1 + ((@temptype@ *)data_out)[0] = ((@temptype@ *)data0)[0] + + ((@temptype@ *)data_out)[0]; + ((@temptype@ *)data_out)[1] = ((@temptype@ *)data0)[1] + + ((@temptype@ *)data_out)[1]; + data0 += stride0; + data_out += stride_out; +# else +# if @nop@ <= 3 +#define _SUMPROD_NOP @nop@ +# else +#define _SUMPROD_NOP nop +# endif + @temptype@ re, im, tmp; + int i; + re = ((@temptype@ *)dataptr[0])[0]; + im = ((@temptype@ *)dataptr[0])[1]; + for (i = 1; i < _SUMPROD_NOP; ++i) { + tmp = re * ((@temptype@ *)dataptr[i])[0] - + im * ((@temptype@ *)dataptr[i])[1]; + im = re * ((@temptype@ *)dataptr[i])[1] + + im * ((@temptype@ *)dataptr[i])[0]; + re = tmp; + } + ((@temptype@ *)dataptr[_SUMPROD_NOP])[0] = re + + ((@temptype@ *)dataptr[_SUMPROD_NOP])[0]; + ((@temptype@ *)dataptr[_SUMPROD_NOP])[1] = im + + ((@temptype@ *)dataptr[_SUMPROD_NOP])[1]; + + for (i = 0; i <= _SUMPROD_NOP; ++i) { + dataptr[i] += strides[i]; + } +#undef _SUMPROD_NOP +# endif +#endif + } +} + +#if @nop@ == 1 + +static void +@name@_sum_of_products_contig_one(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @type@ *data0 = (@type@ *)dataptr[0]; + @type@ *data_out = (@type@ *)dataptr[1]; + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_one (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: +#if !@complex@ + data_out[@i@] = @to@(@from@(data0[@i@]) + + @from@(data_out[@i@])); +#else + ((@temptype@ *)data_out + 2*@i@)[0] = + ((@temptype@ *)data0 + 2*@i@)[0] + + ((@temptype@ *)data_out + 2*@i@)[0]; + ((@temptype@ *)data_out + 2*@i@)[1] = + ((@temptype@ *)data0 + 2*@i@)[1] + + ((@temptype@ *)data_out + 2*@i@)[1]; +#endif +/**end repeat2**/ + case 0: + return; + } + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ +#if !@complex@ + data_out[@i@] = @to@(@from@(data0[@i@]) + + @from@(data_out[@i@])); +#else /* complex */ + ((@temptype@ *)data_out + 2*@i@)[0] = + ((@temptype@ *)data0 + 2*@i@)[0] + + ((@temptype@ *)data_out + 2*@i@)[0]; + ((@temptype@ *)data_out + 2*@i@)[1] = + ((@temptype@ *)data0 + 2*@i@)[1] + + ((@temptype@ *)data_out + 2*@i@)[1]; +#endif +/**end repeat2**/ + data0 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +#elif @nop@ == 2 && !@complex@ + +static void +@name@_sum_of_products_contig_two(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @type@ *data0 = (@type@ *)dataptr[0]; + @type@ *data1 = (@type@ *)dataptr[1]; + @type@ *data_out = (@type@ *)dataptr[2]; + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, b; +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, b; +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_two (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: + data_out[@i@] = @to@(@from@(data0[@i@]) * + @from@(data1[@i@]) + + @from@(data_out[@i@])); +/**end repeat2**/ + case 0: + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1) && + EINSUM_IS_SSE_ALIGNED(data_out)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 4# + */ + a = _mm_mul_ps(_mm_load_ps(data0+@i@), _mm_load_ps(data1+@i@)); + b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); + _mm_store_ps(data_out+@i@, b); +/**end repeat2**/ + data0 += 8; + data1 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#elif EINSUM_USE_SSE2 && @float64@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1) && + EINSUM_IS_SSE_ALIGNED(data_out)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + a = _mm_mul_pd(_mm_load_pd(data0+@i@), _mm_load_pd(data1+@i@)); + b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); + _mm_store_pd(data_out+@i@, b); +/**end repeat2**/ + data0 += 8; + data1 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ +/**begin repeat2 + * #i = 0, 4# + */ + a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), _mm_loadu_ps(data1+@i@)); + b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); + _mm_storeu_ps(data_out+@i@, b); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), _mm_loadu_pd(data1+@i@)); + b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); + _mm_storeu_pd(data_out+@i@, b); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + data_out[@i@] = @to@(@from@(data0[@i@]) * + @from@(data1[@i@]) + + @from@(data_out[@i@])); +/**end repeat2**/ +#endif + data0 += 8; + data1 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +/* Some extra specializations for the two operand case */ +static void +@name@_sum_of_products_stride0_contig_outcontig_two(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @temptype@ value0 = @from@(*(@type@ *)dataptr[0]); + @type@ *data1 = (@type@ *)dataptr[1]; + @type@ *data_out = (@type@ *)dataptr[2]; + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, b, value0_sse; +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, b, value0_sse; +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_stride0_contig_outcontig_two (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: + data_out[@i@] = @to@(value0 * + @from@(data1[@i@]) + + @from@(data_out[@i@])); +/**end repeat2**/ + case 0: + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + value0_sse = _mm_set_ps1(value0); + + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data1) && EINSUM_IS_SSE_ALIGNED(data_out)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 4# + */ + a = _mm_mul_ps(value0_sse, _mm_load_ps(data1+@i@)); + b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); + _mm_store_ps(data_out+@i@, b); +/**end repeat2**/ + data1 += 8; + data_out += 8; + } + + /* Finish off the loop */ + if (count > 0) { + goto finish_after_unrolled_loop; + } + else { + return; + } + } +#elif EINSUM_USE_SSE2 && @float64@ + value0_sse = _mm_set1_pd(value0); + + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data1) && EINSUM_IS_SSE_ALIGNED(data_out)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + a = _mm_mul_pd(value0_sse, _mm_load_pd(data1+@i@)); + b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); + _mm_store_pd(data_out+@i@, b); +/**end repeat2**/ + data1 += 8; + data_out += 8; + } + + /* Finish off the loop */ + if (count > 0) { + goto finish_after_unrolled_loop; + } + else { + return; + } + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ +/**begin repeat2 + * #i = 0, 4# + */ + a = _mm_mul_ps(value0_sse, _mm_loadu_ps(data1+@i@)); + b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); + _mm_storeu_ps(data_out+@i@, b); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + a = _mm_mul_pd(value0_sse, _mm_loadu_pd(data1+@i@)); + b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); + _mm_storeu_pd(data_out+@i@, b); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + data_out[@i@] = @to@(value0 * + @from@(data1[@i@]) + + @from@(data_out[@i@])); +/**end repeat2**/ +#endif + data1 += 8; + data_out += 8; + } + + /* Finish off the loop */ + if (count > 0) { + goto finish_after_unrolled_loop; + } +} + +static void +@name@_sum_of_products_contig_stride0_outcontig_two(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @type@ *data0 = (@type@ *)dataptr[0]; + @temptype@ value1 = @from@(*(@type@ *)dataptr[1]); + @type@ *data_out = (@type@ *)dataptr[2]; + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, b, value1_sse; +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, b, value1_sse; +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_stride0_outcontig_two (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: + data_out[@i@] = @to@(@from@(data0[@i@])* + value1 + + @from@(data_out[@i@])); +/**end repeat2**/ + case 0: + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + value1_sse = _mm_set_ps1(value1); + + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data_out)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 4# + */ + a = _mm_mul_ps(_mm_load_ps(data0+@i@), value1_sse); + b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); + _mm_store_ps(data_out+@i@, b); +/**end repeat2**/ + data0 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#elif EINSUM_USE_SSE2 && @float64@ + value1_sse = _mm_set1_pd(value1); + + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data_out)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + a = _mm_mul_pd(_mm_load_pd(data0+@i@), value1_sse); + b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); + _mm_store_pd(data_out+@i@, b); +/**end repeat2**/ + data0 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ +/**begin repeat2 + * #i = 0, 4# + */ + a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), value1_sse); + b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); + _mm_storeu_ps(data_out+@i@, b); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), value1_sse); + b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); + _mm_storeu_pd(data_out+@i@, b); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + data_out[@i@] = @to@(@from@(data0[@i@])* + value1 + + @from@(data_out[@i@])); +/**end repeat2**/ +#endif + data0 += 8; + data_out += 8; + } + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +static void +@name@_sum_of_products_contig_contig_outstride0_two(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @type@ *data0 = (@type@ *)dataptr[0]; + @type@ *data1 = (@type@ *)dataptr[1]; + @temptype@ accum = 0; + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, accum_sse = _mm_setzero_ps(); +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, accum_sse = _mm_setzero_pd(); +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_contig_outstride0_two (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: + accum += @from@(data0[@i@]) * @from@(data1[@i@]); +/**end repeat2**/ + case 0: + *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum); + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + + _mm_prefetch(data0 + 512, _MM_HINT_T0); + _mm_prefetch(data1 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + a = _mm_mul_ps(_mm_load_ps(data0+@i@), _mm_load_ps(data1+@i@)); + accum_sse = _mm_add_ps(accum_sse, a); +/**end repeat2**/ + data0 += 8; + data1 += 8; + } + + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#elif EINSUM_USE_SSE2 && @float64@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + + _mm_prefetch(data0 + 512, _MM_HINT_T0); + _mm_prefetch(data1 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + a = _mm_mul_pd(_mm_load_pd(data0+@i@), _mm_load_pd(data1+@i@)); + accum_sse = _mm_add_pd(accum_sse, a); +/**end repeat2**/ + data0 += 8; + data1 += 8; + } + + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ + _mm_prefetch(data0 + 512, _MM_HINT_T0); + _mm_prefetch(data1 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), _mm_loadu_ps(data1+@i@)); + accum_sse = _mm_add_ps(accum_sse, a); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ + _mm_prefetch(data0 + 512, _MM_HINT_T0); + _mm_prefetch(data1 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), _mm_loadu_pd(data1+@i@)); + accum_sse = _mm_add_pd(accum_sse, a); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + accum += @from@(data0[@i@]) * @from@(data1[@i@]); +/**end repeat2**/ +#endif + data0 += 8; + data1 += 8; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); +#elif EINSUM_USE_SSE2 && @float64@ + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); +#endif + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +static void +@name@_sum_of_products_stride0_contig_outstride0_two(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @temptype@ value0 = @from@(*(@type@ *)dataptr[0]); + @type@ *data1 = (@type@ *)dataptr[1]; + @temptype@ accum = 0; + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, accum_sse = _mm_setzero_ps(); +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, accum_sse = _mm_setzero_pd(); +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_stride0_contig_outstride0_two (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: + accum += @from@(data1[@i@]); +/**end repeat2**/ + case 0: + *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + value0 * accum); + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data1)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data1+@i@)); +/**end repeat2**/ + data1 += 8; + } + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#elif EINSUM_USE_SSE2 && @float64@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data1)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data1+@i@)); +/**end repeat2**/ + data1 += 8; + } + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data1+@i@)); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data1+@i@)); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + accum += @from@(data1[@i@]); +/**end repeat2**/ +#endif + data1 += 8; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); +#elif EINSUM_USE_SSE2 && @float64@ + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); +#endif + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +static void +@name@_sum_of_products_contig_stride0_outstride0_two(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @type@ *data0 = (@type@ *)dataptr[0]; + @temptype@ value1 = @from@(*(@type@ *)dataptr[1]); + @temptype@ accum = 0; + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, accum_sse = _mm_setzero_ps(); +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, accum_sse = _mm_setzero_pd(); +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_stride0_outstride0_two (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: + accum += @from@(data0[@i@]); +/**end repeat2**/ + case 0: + *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum * value1); + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data0+@i@)); +/**end repeat2**/ + data0 += 8; + } + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#elif EINSUM_USE_SSE2 && @float64@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data0+@i@)); +/**end repeat2**/ + data0 += 8; + } + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data0+@i@)); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data0+@i@)); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + accum += @from@(data0[@i@]); +/**end repeat2**/ +#endif + data0 += 8; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); +#elif EINSUM_USE_SSE2 && @float64@ + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); +#endif + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +#elif @nop@ == 3 && !@complex@ + +static void +@name@_sum_of_products_contig_three(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + @type@ *data0 = (@type@ *)dataptr[0]; + @type@ *data1 = (@type@ *)dataptr[1]; + @type@ *data2 = (@type@ *)dataptr[2]; + @type@ *data_out = (@type@ *)dataptr[3]; + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + data_out[@i@] = @to@(@from@(data0[@i@]) * + @from@(data1[@i@]) * + @from@(data2[@i@]) + + @from@(data_out[@i@])); +/**end repeat2**/ + data0 += 8; + data1 += 8; + data2 += 8; + data_out += 8; + } + + /* Finish off the loop */ + +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + if (count-- == 0) { + return; + } + data_out[@i@] = @to@(@from@(data0[@i@]) * + @from@(data1[@i@]) * + @from@(data2[@i@]) + + @from@(data_out[@i@])); +/**end repeat2**/ +} + +#else /* @nop@ > 3 || @complex */ + +static void +@name@_sum_of_products_contig_@noplabel@(int nop, char **dataptr, + npy_intp const *NPY_UNUSED(strides), npy_intp count) +{ + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_@noplabel@ (%d)\n", + (int)count); + + while (count--) { +#if !@complex@ + @temptype@ temp = @from@(*(@type@ *)dataptr[0]); + int i; + for (i = 1; i < nop; ++i) { + temp *= @from@(*(@type@ *)dataptr[i]); + } + *(@type@ *)dataptr[nop] = @to@(temp + + @from@(*(@type@ *)dataptr[i])); + for (i = 0; i <= nop; ++i) { + dataptr[i] += sizeof(@type@); + } +#else /* complex */ +# if @nop@ <= 3 +# define _SUMPROD_NOP @nop@ +# else +# define _SUMPROD_NOP nop +# endif + @temptype@ re, im, tmp; + int i; + re = ((@temptype@ *)dataptr[0])[0]; + im = ((@temptype@ *)dataptr[0])[1]; + for (i = 1; i < _SUMPROD_NOP; ++i) { + tmp = re * ((@temptype@ *)dataptr[i])[0] - + im * ((@temptype@ *)dataptr[i])[1]; + im = re * ((@temptype@ *)dataptr[i])[1] + + im * ((@temptype@ *)dataptr[i])[0]; + re = tmp; + } + ((@temptype@ *)dataptr[_SUMPROD_NOP])[0] = re + + ((@temptype@ *)dataptr[_SUMPROD_NOP])[0]; + ((@temptype@ *)dataptr[_SUMPROD_NOP])[1] = im + + ((@temptype@ *)dataptr[_SUMPROD_NOP])[1]; + + for (i = 0; i <= _SUMPROD_NOP; ++i) { + dataptr[i] += sizeof(@type@); + } +# undef _SUMPROD_NOP +#endif + } +} + +#endif /* functions for various @nop@ */ + +#if @nop@ == 1 + +static void +@name@_sum_of_products_contig_outstride0_one(int nop, char **dataptr, + npy_intp const *strides, npy_intp count) +{ +#if @complex@ + @temptype@ accum_re = 0, accum_im = 0; + @temptype@ *data0 = (@temptype@ *)dataptr[0]; +#else + @temptype@ accum = 0; + @type@ *data0 = (@type@ *)dataptr[0]; +#endif + +#if EINSUM_USE_SSE1 && @float32@ + __m128 a, accum_sse = _mm_setzero_ps(); +#elif EINSUM_USE_SSE2 && @float64@ + __m128d a, accum_sse = _mm_setzero_pd(); +#endif + + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_outstride0_one (%d)\n", + (int)count); + +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat2 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: +#if !@complex@ + accum += @from@(data0[@i@]); +#else /* complex */ + accum_re += data0[2*@i@+0]; + accum_im += data0[2*@i@+1]; +#endif +/**end repeat2**/ + case 0: +#if @complex@ + ((@temptype@ *)dataptr[1])[0] += accum_re; + ((@temptype@ *)dataptr[1])[1] += accum_im; +#else + *((@type@ *)dataptr[1]) = @to@(accum + + @from@(*((@type@ *)dataptr[1]))); +#endif + return; + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + + _mm_prefetch(data0 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data0+@i@)); +/**end repeat2**/ + data0 += 8; + } + + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#elif EINSUM_USE_SSE2 && @float64@ + /* Use aligned instructions if possible */ + if (EINSUM_IS_SSE_ALIGNED(data0)) { + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + + _mm_prefetch(data0 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data0+@i@)); +/**end repeat2**/ + data0 += 8; + } + + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); + + /* Finish off the loop */ + goto finish_after_unrolled_loop; + } +#endif + + /* Unroll the loop by 8 */ + while (count >= 8) { + count -= 8; + +#if EINSUM_USE_SSE1 && @float32@ + _mm_prefetch(data0 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 4# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data0+@i@)); +/**end repeat2**/ +#elif EINSUM_USE_SSE2 && @float64@ + _mm_prefetch(data0 + 512, _MM_HINT_T0); + +/**begin repeat2 + * #i = 0, 2, 4, 6# + */ + /* + * NOTE: This accumulation changes the order, so will likely + * produce slightly different results. + */ + accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data0+@i@)); +/**end repeat2**/ +#else +/**begin repeat2 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ +# if !@complex@ + accum += @from@(data0[@i@]); +# else /* complex */ + accum_re += data0[2*@i@+0]; + accum_im += data0[2*@i@+1]; +# endif +/**end repeat2**/ +#endif + +#if !@complex@ + data0 += 8; +#else + data0 += 8*2; +#endif + } + +#if EINSUM_USE_SSE1 && @float32@ + /* Add the four SSE values and put in accum */ + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); + accum_sse = _mm_add_ps(a, accum_sse); + a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); + accum_sse = _mm_add_ps(a, accum_sse); + _mm_store_ss(&accum, accum_sse); +#elif EINSUM_USE_SSE2 && @float64@ + /* Add the two SSE2 values and put in accum */ + a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); + accum_sse = _mm_add_pd(a, accum_sse); + _mm_store_sd(&accum, accum_sse); +#endif + + /* Finish off the loop */ + goto finish_after_unrolled_loop; +} + +#endif /* @nop@ == 1 */ + +static void +@name@_sum_of_products_outstride0_@noplabel@(int nop, char **dataptr, + npy_intp const *strides, npy_intp count) +{ +#if @complex@ + @temptype@ accum_re = 0, accum_im = 0; +#else + @temptype@ accum = 0; +#endif + +#if (@nop@ == 1) || (@nop@ <= 3 && !@complex@) + char *data0 = dataptr[0]; + npy_intp stride0 = strides[0]; +#endif +#if (@nop@ == 2 || @nop@ == 3) && !@complex@ + char *data1 = dataptr[1]; + npy_intp stride1 = strides[1]; +#endif +#if (@nop@ == 3) && !@complex@ + char *data2 = dataptr[2]; + npy_intp stride2 = strides[2]; +#endif + + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_outstride0_@noplabel@ (%d)\n", + (int)count); + + while (count--) { +#if !@complex@ +# if @nop@ == 1 + accum += @from@(*(@type@ *)data0); + data0 += stride0; +# elif @nop@ == 2 + accum += @from@(*(@type@ *)data0) * + @from@(*(@type@ *)data1); + data0 += stride0; + data1 += stride1; +# elif @nop@ == 3 + accum += @from@(*(@type@ *)data0) * + @from@(*(@type@ *)data1) * + @from@(*(@type@ *)data2); + data0 += stride0; + data1 += stride1; + data2 += stride2; +# else + @temptype@ temp = @from@(*(@type@ *)dataptr[0]); + int i; + for (i = 1; i < nop; ++i) { + temp *= @from@(*(@type@ *)dataptr[i]); + } + accum += temp; + for (i = 0; i < nop; ++i) { + dataptr[i] += strides[i]; + } +# endif +#else /* complex */ +# if @nop@ == 1 + accum_re += ((@temptype@ *)data0)[0]; + accum_im += ((@temptype@ *)data0)[1]; + data0 += stride0; +# else +# if @nop@ <= 3 +#define _SUMPROD_NOP @nop@ +# else +#define _SUMPROD_NOP nop +# endif + @temptype@ re, im, tmp; + int i; + re = ((@temptype@ *)dataptr[0])[0]; + im = ((@temptype@ *)dataptr[0])[1]; + for (i = 1; i < _SUMPROD_NOP; ++i) { + tmp = re * ((@temptype@ *)dataptr[i])[0] - + im * ((@temptype@ *)dataptr[i])[1]; + im = re * ((@temptype@ *)dataptr[i])[1] + + im * ((@temptype@ *)dataptr[i])[0]; + re = tmp; + } + accum_re += re; + accum_im += im; + for (i = 0; i < _SUMPROD_NOP; ++i) { + dataptr[i] += strides[i]; + } +#undef _SUMPROD_NOP +# endif +#endif + } + +#if @complex@ +# if @nop@ <= 3 + ((@temptype@ *)dataptr[@nop@])[0] += accum_re; + ((@temptype@ *)dataptr[@nop@])[1] += accum_im; +# else + ((@temptype@ *)dataptr[nop])[0] += accum_re; + ((@temptype@ *)dataptr[nop])[1] += accum_im; +# endif +#else +# if @nop@ <= 3 + *((@type@ *)dataptr[@nop@]) = @to@(accum + + @from@(*((@type@ *)dataptr[@nop@]))); +# else + *((@type@ *)dataptr[nop]) = @to@(accum + + @from@(*((@type@ *)dataptr[nop]))); +# endif +#endif + +} + +/**end repeat1**/ + +/**end repeat**/ + + +/* Do OR of ANDs for the boolean type */ + +/**begin repeat + * #nop = 1, 2, 3, 1000# + * #noplabel = one, two, three, any# + */ + +static void +bool_sum_of_products_@noplabel@(int nop, char **dataptr, + npy_intp const *strides, npy_intp count) +{ +#if (@nop@ <= 3) + char *data0 = dataptr[0]; + npy_intp stride0 = strides[0]; +#endif +#if (@nop@ == 2 || @nop@ == 3) + char *data1 = dataptr[1]; + npy_intp stride1 = strides[1]; +#endif +#if (@nop@ == 3) + char *data2 = dataptr[2]; + npy_intp stride2 = strides[2]; +#endif +#if (@nop@ <= 3) + char *data_out = dataptr[@nop@]; + npy_intp stride_out = strides[@nop@]; +#endif + + while (count--) { +#if @nop@ == 1 + *(npy_bool *)data_out = *(npy_bool *)data0 || + *(npy_bool *)data_out; + data0 += stride0; + data_out += stride_out; +#elif @nop@ == 2 + *(npy_bool *)data_out = (*(npy_bool *)data0 && + *(npy_bool *)data1) || + *(npy_bool *)data_out; + data0 += stride0; + data1 += stride1; + data_out += stride_out; +#elif @nop@ == 3 + *(npy_bool *)data_out = (*(npy_bool *)data0 && + *(npy_bool *)data1 && + *(npy_bool *)data2) || + *(npy_bool *)data_out; + data0 += stride0; + data1 += stride1; + data2 += stride2; + data_out += stride_out; +#else + npy_bool temp = *(npy_bool *)dataptr[0]; + int i; + for (i = 1; i < nop; ++i) { + temp = temp && *(npy_bool *)dataptr[i]; + } + *(npy_bool *)dataptr[nop] = temp || *(npy_bool *)dataptr[i]; + for (i = 0; i <= nop; ++i) { + dataptr[i] += strides[i]; + } +#endif + } +} + +static void +bool_sum_of_products_contig_@noplabel@(int nop, char **dataptr, + npy_intp const *strides, npy_intp count) +{ +#if (@nop@ <= 3) + char *data0 = dataptr[0]; +#endif +#if (@nop@ == 2 || @nop@ == 3) + char *data1 = dataptr[1]; +#endif +#if (@nop@ == 3) + char *data2 = dataptr[2]; +#endif +#if (@nop@ <= 3) + char *data_out = dataptr[@nop@]; +#endif + +#if (@nop@ <= 3) +/* This is placed before the main loop to make small counts faster */ +finish_after_unrolled_loop: + switch (count) { +/**begin repeat1 + * #i = 6, 5, 4, 3, 2, 1, 0# + */ + case @i@+1: +# if @nop@ == 1 + ((npy_bool *)data_out)[@i@] = ((npy_bool *)data0)[@i@] || + ((npy_bool *)data_out)[@i@]; +# elif @nop@ == 2 + ((npy_bool *)data_out)[@i@] = + (((npy_bool *)data0)[@i@] && + ((npy_bool *)data1)[@i@]) || + ((npy_bool *)data_out)[@i@]; +# elif @nop@ == 3 + ((npy_bool *)data_out)[@i@] = + (((npy_bool *)data0)[@i@] && + ((npy_bool *)data1)[@i@] && + ((npy_bool *)data2)[@i@]) || + ((npy_bool *)data_out)[@i@]; +# endif +/**end repeat1**/ + case 0: + return; + } +#endif + +/* Unroll the loop by 8 for fixed-size nop */ +#if (@nop@ <= 3) + while (count >= 8) { + count -= 8; +#else + while (count--) { +#endif + +# if @nop@ == 1 +/**begin repeat1 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + *((npy_bool *)data_out + @i@) = (*((npy_bool *)data0 + @i@)) || + (*((npy_bool *)data_out + @i@)); +/**end repeat1**/ + data0 += 8*sizeof(npy_bool); + data_out += 8*sizeof(npy_bool); +# elif @nop@ == 2 +/**begin repeat1 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + *((npy_bool *)data_out + @i@) = + ((*((npy_bool *)data0 + @i@)) && + (*((npy_bool *)data1 + @i@))) || + (*((npy_bool *)data_out + @i@)); +/**end repeat1**/ + data0 += 8*sizeof(npy_bool); + data1 += 8*sizeof(npy_bool); + data_out += 8*sizeof(npy_bool); +# elif @nop@ == 3 +/**begin repeat1 + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + *((npy_bool *)data_out + @i@) = + ((*((npy_bool *)data0 + @i@)) && + (*((npy_bool *)data1 + @i@)) && + (*((npy_bool *)data2 + @i@))) || + (*((npy_bool *)data_out + @i@)); +/**end repeat1**/ + data0 += 8*sizeof(npy_bool); + data1 += 8*sizeof(npy_bool); + data2 += 8*sizeof(npy_bool); + data_out += 8*sizeof(npy_bool); +# else + npy_bool temp = *(npy_bool *)dataptr[0]; + int i; + for (i = 1; i < nop; ++i) { + temp = temp && *(npy_bool *)dataptr[i]; + } + *(npy_bool *)dataptr[nop] = temp || *(npy_bool *)dataptr[i]; + for (i = 0; i <= nop; ++i) { + dataptr[i] += sizeof(npy_bool); + } +# endif + } + + /* If the loop was unrolled, we need to finish it off */ +#if (@nop@ <= 3) + goto finish_after_unrolled_loop; +#endif +} + +static void +bool_sum_of_products_outstride0_@noplabel@(int nop, char **dataptr, + npy_intp const *strides, npy_intp count) +{ + npy_bool accum = 0; + +#if (@nop@ <= 3) + char *data0 = dataptr[0]; + npy_intp stride0 = strides[0]; +#endif +#if (@nop@ == 2 || @nop@ == 3) + char *data1 = dataptr[1]; + npy_intp stride1 = strides[1]; +#endif +#if (@nop@ == 3) + char *data2 = dataptr[2]; + npy_intp stride2 = strides[2]; +#endif + + while (count--) { +#if @nop@ == 1 + accum = *(npy_bool *)data0 || accum; + data0 += stride0; +#elif @nop@ == 2 + accum = (*(npy_bool *)data0 && *(npy_bool *)data1) || accum; + data0 += stride0; + data1 += stride1; +#elif @nop@ == 3 + accum = (*(npy_bool *)data0 && + *(npy_bool *)data1 && + *(npy_bool *)data2) || accum; + data0 += stride0; + data1 += stride1; + data2 += stride2; +#else + npy_bool temp = *(npy_bool *)dataptr[0]; + int i; + for (i = 1; i < nop; ++i) { + temp = temp && *(npy_bool *)dataptr[i]; + } + accum = temp || accum; + for (i = 0; i <= nop; ++i) { + dataptr[i] += strides[i]; + } +#endif + } + +# if @nop@ <= 3 + *((npy_bool *)dataptr[@nop@]) = accum || *((npy_bool *)dataptr[@nop@]); +# else + *((npy_bool *)dataptr[nop]) = accum || *((npy_bool *)dataptr[nop]); +# endif +} + +/**end repeat**/ + +/* These tables need to match up with the type enum */ +static sum_of_products_fn +_contig_outstride0_unary_specialization_table[NPY_NTYPES] = { +/**begin repeat + * #name = bool, + * byte, ubyte, + * short, ushort, + * int, uint, + * long, ulong, + * longlong, ulonglong, + * float, double, longdouble, + * cfloat, cdouble, clongdouble, + * object, string, unicode, void, + * datetime, timedelta, half# + * #use = 0, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, 1, + * 1, 1, 1, + * 0, 0, 0, 0, + * 0, 0, 1# + */ +#if @use@ + &@name@_sum_of_products_contig_outstride0_one, +#else + NULL, +#endif +/**end repeat**/ +}; /* End of _contig_outstride0_unary_specialization_table */ + +static sum_of_products_fn _binary_specialization_table[NPY_NTYPES][5] = { +/**begin repeat + * #name = bool, + * byte, ubyte, + * short, ushort, + * int, uint, + * long, ulong, + * longlong, ulonglong, + * float, double, longdouble, + * cfloat, cdouble, clongdouble, + * object, string, unicode, void, + * datetime, timedelta, half# + * #use = 0, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, 1, + * 0, 0, 0, + * 0, 0, 0, 0, + * 0, 0, 1# + */ +#if @use@ +{ + &@name@_sum_of_products_stride0_contig_outstride0_two, + &@name@_sum_of_products_stride0_contig_outcontig_two, + &@name@_sum_of_products_contig_stride0_outstride0_two, + &@name@_sum_of_products_contig_stride0_outcontig_two, + &@name@_sum_of_products_contig_contig_outstride0_two, +}, +#else + {NULL, NULL, NULL, NULL, NULL}, +#endif +/**end repeat**/ +}; /* End of _binary_specialization_table */ + +static sum_of_products_fn _outstride0_specialized_table[NPY_NTYPES][4] = { +/**begin repeat + * #name = bool, + * byte, ubyte, + * short, ushort, + * int, uint, + * long, ulong, + * longlong, ulonglong, + * float, double, longdouble, + * cfloat, cdouble, clongdouble, + * object, string, unicode, void, + * datetime, timedelta, half# + * #use = 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, 1, + * 1, 1, 1, + * 0, 0, 0, 0, + * 0, 0, 1# + */ +#if @use@ +{ + &@name@_sum_of_products_outstride0_any, + &@name@_sum_of_products_outstride0_one, + &@name@_sum_of_products_outstride0_two, + &@name@_sum_of_products_outstride0_three +}, +#else + {NULL, NULL, NULL, NULL}, +#endif +/**end repeat**/ +}; /* End of _outstride0_specialized_table */ + +static sum_of_products_fn _allcontig_specialized_table[NPY_NTYPES][4] = { +/**begin repeat + * #name = bool, + * byte, ubyte, + * short, ushort, + * int, uint, + * long, ulong, + * longlong, ulonglong, + * float, double, longdouble, + * cfloat, cdouble, clongdouble, + * object, string, unicode, void, + * datetime, timedelta, half# + * #use = 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, 1, + * 1, 1, 1, + * 0, 0, 0, 0, + * 0, 0, 1# + */ +#if @use@ +{ + &@name@_sum_of_products_contig_any, + &@name@_sum_of_products_contig_one, + &@name@_sum_of_products_contig_two, + &@name@_sum_of_products_contig_three +}, +#else + {NULL, NULL, NULL, NULL}, +#endif +/**end repeat**/ +}; /* End of _allcontig_specialized_table */ + +static sum_of_products_fn _unspecialized_table[NPY_NTYPES][4] = { +/**begin repeat + * #name = bool, + * byte, ubyte, + * short, ushort, + * int, uint, + * long, ulong, + * longlong, ulonglong, + * float, double, longdouble, + * cfloat, cdouble, clongdouble, + * object, string, unicode, void, + * datetime, timedelta, half# + * #use = 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, + * 1, 1, 1, + * 1, 1, 1, + * 0, 0, 0, 0, + * 0, 0, 1# + */ +#if @use@ +{ + &@name@_sum_of_products_any, + &@name@_sum_of_products_one, + &@name@_sum_of_products_two, + &@name@_sum_of_products_three +}, +#else + {NULL, NULL, NULL, NULL}, +#endif +/**end repeat**/ +}; /* End of _unnspecialized_table */ + +NPY_VISIBILITY_HIDDEN sum_of_products_fn +get_sum_of_products_function(int nop, int type_num, + npy_intp itemsize, npy_intp const *fixed_strides) +{ + int iop; + + if (type_num >= NPY_NTYPES) { + return NULL; + } + + /* contiguous reduction */ + if (nop == 1 && fixed_strides[0] == itemsize && fixed_strides[1] == 0) { + sum_of_products_fn ret = + _contig_outstride0_unary_specialization_table[type_num]; + if (ret != NULL) { + return ret; + } + } + + /* nop of 2 has more specializations */ + if (nop == 2) { + /* Encode the zero/contiguous strides */ + int code; + code = (fixed_strides[0] == 0) ? 0 : + (fixed_strides[0] == itemsize) ? 2*2*1 : 8; + code += (fixed_strides[1] == 0) ? 0 : + (fixed_strides[1] == itemsize) ? 2*1 : 8; + code += (fixed_strides[2] == 0) ? 0 : + (fixed_strides[2] == itemsize) ? 1 : 8; + if (code >= 2 && code < 7) { + sum_of_products_fn ret = + _binary_specialization_table[type_num][code-2]; + if (ret != NULL) { + return ret; + } + } + } + + /* Inner loop with an output stride of 0 */ + if (fixed_strides[nop] == 0) { + return _outstride0_specialized_table[type_num][nop <= 3 ? nop : 0]; + } + + /* Check for all contiguous */ + for (iop = 0; iop < nop + 1; ++iop) { + if (fixed_strides[iop] != itemsize) { + break; + } + } + + /* Contiguous loop */ + if (iop == nop + 1) { + return _allcontig_specialized_table[type_num][nop <= 3 ? nop : 0]; + } + + /* None of the above specializations caught it, general loops */ + return _unspecialized_table[type_num][nop <= 3 ? nop : 0]; +} diff --git a/numpy/core/src/multiarray/einsum_sumprod.h b/numpy/core/src/multiarray/einsum_sumprod.h new file mode 100644 index 000000000..c6cf18ec6 --- /dev/null +++ b/numpy/core/src/multiarray/einsum_sumprod.h @@ -0,0 +1,12 @@ +#ifndef _NPY_MULTIARRAY_EINSUM_SUMPROD_H +#define _NPY_MULTIARRAY_EINSUM_SUMPROD_H + +#include + +typedef void (*sum_of_products_fn)(int, char **, npy_intp const*, npy_intp); + +NPY_VISIBILITY_HIDDEN sum_of_products_fn +get_sum_of_products_function(int nop, int type_num, + npy_intp itemsize, npy_intp const *fixed_strides); + +#endif -- cgit v1.2.1 From 4eae213d4bbe1b509ceff897db8f65be9c480d14 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Thu, 20 Aug 2020 11:30:50 +0100 Subject: DOC: Fix spacing in vectorize doc Fix spacing that produced incorrect rendering --- numpy/lib/function_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 556227c0d..0db00a0f2 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1991,8 +1991,8 @@ class vectorize: .. versionadded:: 1.7.0 cache : bool, optional - If `True`, then cache the first function call that determines the number - of outputs if `otypes` is not provided. + If `True`, then cache the first function call that determines the number + of outputs if `otypes` is not provided. .. versionadded:: 1.7.0 -- cgit v1.2.1 From 6d33a659b493b43b393e2ce58cbb9e8e40e2f583 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 13:50:28 +0200 Subject: REV: Removed `compat` and `core` stubs --- numpy/compat/__init__.pyi | 26 --- numpy/core/__init__.pyi | 425 ---------------------------------------------- 2 files changed, 451 deletions(-) delete mode 100644 numpy/compat/__init__.pyi delete mode 100644 numpy/core/__init__.pyi diff --git a/numpy/compat/__init__.pyi b/numpy/compat/__init__.pyi deleted file mode 100644 index ff55ccfde..000000000 --- a/numpy/compat/__init__.pyi +++ /dev/null @@ -1,26 +0,0 @@ -from typing import Any - -getargspec: Any -formatargspec: Any -bytes: Any -asbytes: Any -isfileobj: Any -getexception: Any -strchar: Any -unicode: Any -asunicode: Any -asbytes_nested: Any -asunicode_nested: Any -asstr: Any -open_latin1: Any -long: Any -basestring: Any -sixu: Any -integer_types: Any -is_pathlib_path: Any -npy_load_module: Any -Path: Any -pickle: Any -contextlib_nullcontext: Any -os_fspath: Any -os_PathLike: Any diff --git a/numpy/core/__init__.pyi b/numpy/core/__init__.pyi deleted file mode 100644 index 9e3cf3d75..000000000 --- a/numpy/core/__init__.pyi +++ /dev/null @@ -1,425 +0,0 @@ -from typing import Any - -char: Any -rec: Any -memmap: Any -newaxis: Any -ndarray: Any -flatiter: Any -nditer: Any -nested_iters: Any -ufunc: Any -arange: Any -array: Any -zeros: Any -count_nonzero: Any -empty: Any -broadcast: Any -dtype: Any -fromstring: Any -fromfile: Any -frombuffer: Any -where: Any -argwhere: Any -copyto: Any -concatenate: Any -fastCopyAndTranspose: Any -lexsort: Any -set_numeric_ops: Any -can_cast: Any -promote_types: Any -min_scalar_type: Any -result_type: Any -isfortran: Any -empty_like: Any -zeros_like: Any -ones_like: Any -correlate: Any -convolve: Any -inner: Any -dot: Any -outer: Any -vdot: Any -roll: Any -rollaxis: Any -moveaxis: Any -cross: Any -tensordot: Any -little_endian: Any -fromiter: Any -array_equal: Any -array_equiv: Any -indices: Any -fromfunction: Any -isclose: Any -isscalar: Any -binary_repr: Any -base_repr: Any -ones: Any -identity: Any -allclose: Any -compare_chararrays: Any -putmask: Any -flatnonzero: Any -Inf: Any -inf: Any -infty: Any -Infinity: Any -nan: Any -NaN: Any -False_: Any -True_: Any -bitwise_not: Any -CLIP: Any -RAISE: Any -WRAP: Any -MAXDIMS: Any -BUFSIZE: Any -ALLOW_THREADS: Any -ComplexWarning: Any -full: Any -full_like: Any -matmul: Any -shares_memory: Any -may_share_memory: Any -MAY_SHARE_BOUNDS: Any -MAY_SHARE_EXACT: Any -TooHardError: Any -AxisError: Any -alen: Any -all: Any -alltrue: Any -amax: Any -amin: Any -any: Any -argmax: Any -argmin: Any -argpartition: Any -argsort: Any -around: Any -choose: Any -clip: Any -compress: Any -cumprod: Any -cumproduct: Any -cumsum: Any -diagonal: Any -mean: Any -ndim: Any -nonzero: Any -partition: Any -prod: Any -product: Any -ptp: Any -put: Any -ravel: Any -repeat: Any -reshape: Any -resize: Any -round_: Any -searchsorted: Any -shape: Any -size: Any -sometrue: Any -sort: Any -squeeze: Any -std: Any -sum: Any -swapaxes: Any -take: Any -trace: Any -transpose: Any -var: Any -_UFUNC_API: Any -ERR_CALL: Any -ERR_DEFAULT: Any -ERR_IGNORE: Any -ERR_LOG: Any -ERR_PRINT: Any -ERR_RAISE: Any -ERR_WARN: Any -FLOATING_POINT_SUPPORT: Any -FPE_DIVIDEBYZERO: Any -FPE_INVALID: Any -FPE_OVERFLOW: Any -FPE_UNDERFLOW: Any -NAN: Any -NINF: Any -NZERO: Any -PINF: Any -PZERO: Any -SHIFT_DIVIDEBYZERO: Any -SHIFT_INVALID: Any -SHIFT_OVERFLOW: Any -SHIFT_UNDERFLOW: Any -UFUNC_BUFSIZE_DEFAULT: Any -UFUNC_PYVALS_NAME: Any -_add_newdoc_ufunc: Any -absolute: Any -add: Any -arccos: Any -arccosh: Any -arcsin: Any -arcsinh: Any -arctan: Any -arctan2: Any -arctanh: Any -bitwise_and: Any -bitwise_or: Any -bitwise_xor: Any -cbrt: Any -ceil: Any -conj: Any -conjugate: Any -copysign: Any -cos: Any -cosh: Any -deg2rad: Any -degrees: Any -divide: Any -divmod: Any -e: Any -equal: Any -euler_gamma: Any -exp: Any -exp2: Any -expm1: Any -fabs: Any -floor: Any -floor_divide: Any -float_power: Any -fmax: Any -fmin: Any -fmod: Any -frexp: Any -frompyfunc: Any -gcd: Any -geterrobj: Any -greater: Any -greater_equal: Any -heaviside: Any -hypot: Any -invert: Any -isfinite: Any -isinf: Any -isnan: Any -isnat: Any -lcm: Any -ldexp: Any -left_shift: Any -less: Any -less_equal: Any -log: Any -log10: Any -log1p: Any -log2: Any -logaddexp: Any -logaddexp2: Any -logical_and: Any -logical_not: Any -logical_or: Any -logical_xor: Any -maximum: Any -minimum: Any -mod: Any -modf: Any -multiply: Any -negative: Any -nextafter: Any -not_equal: Any -pi: Any -positive: Any -power: Any -rad2deg: Any -radians: Any -reciprocal: Any -remainder: Any -right_shift: Any -rint: Any -seterrobj: Any -sign: Any -signbit: Any -sin: Any -sinh: Any -spacing: Any -sqrt: Any -square: Any -subtract: Any -tan: Any -tanh: Any -true_divide: Any -trunc: Any -sctypeDict: Any -typeDict: Any -sctypes: Any -ScalarType: Any -obj2sctype: Any -cast: Any -nbytes: Any -sctype2char: Any -maximum_sctype: Any -issctype: Any -typecodes: Any -find_common_type: Any -issubdtype: Any -datetime_data: Any -datetime_as_string: Any -busday_offset: Any -busday_count: Any -is_busday: Any -busdaycalendar: Any -byte: Any -ubyte: Any -short: Any -ushort: Any -uint: Any -intp: Any -uintp: Any -long: Any -longlong: Any -ulonglong: Any -half: Any -double: Any -longdouble: Any -cfloat: Any -cdouble: Any -clongdouble: Any -unicode: Any -void: Any -generic: Any -number: Any -integer: Any -inexact: Any -signedinteger: Any -unsignedinteger: Any -floating: Any -complexfloating: Any -flexible: Any -character: Any -bool8: Any -int64: Any -uint64: Any -float16: Any -float32: Any -float64: Any -complex64: Any -complex128: Any -object0: Any -bytes0: Any -str0: Any -void0: Any -datetime64: Any -timedelta64: Any -Bytes0: Any -Datetime64: Any -Str0: Any -Uint64: Any -int32: Any -uint32: Any -int16: Any -uint16: Any -int8: Any -uint8: Any -complex_: Any -int0: Any -uint0: Any -single: Any -csingle: Any -singlecomplex: Any -float_: Any -intc: Any -uintc: Any -int_: Any -longfloat: Any -clongfloat: Any -longcomplex: Any -bool_: Any -bytes_: Any -string_: Any -str_: Any -unicode_: Any -object_: Any -array2string: Any -array_str: Any -array_repr: Any -set_string_function: Any -set_printoptions: Any -get_printoptions: Any -printoptions: Any -format_float_positional: Any -format_float_scientific: Any -asarray: Any -asanyarray: Any -ascontiguousarray: Any -asfortranarray: Any -require: Any -seterr: Any -geterr: Any -setbufsize: Any -getbufsize: Any -seterrcall: Any -geterrcall: Any -errstate: Any -alen: Any -all: Any -alltrue: Any -amax: Any -amin: Any -any: Any -argmax: Any -argmin: Any -argpartition: Any -argsort: Any -around: Any -choose: Any -clip: Any -compress: Any -cumprod: Any -cumproduct: Any -cumsum: Any -diagonal: Any -mean: Any -ndim: Any -nonzero: Any -partition: Any -prod: Any -product: Any -ptp: Any -put: Any -ravel: Any -repeat: Any -reshape: Any -resize: Any -round_: Any -searchsorted: Any -shape: Any -size: Any -sometrue: Any -sort: Any -squeeze: Any -std: Any -sum: Any -swapaxes: Any -take: Any -trace: Any -transpose: Any -var: Any -record: Any -recarray: Any -format_parser: Any -chararray: Any -logspace: Any -linspace: Any -geomspace: Any -MachAr: Any -finfo: Any -iinfo: Any -atleast_1d: Any -atleast_2d: Any -atleast_3d: Any -block: Any -hstack: Any -stack: Any -vstack: Any -einsum: Any -einsum_path: Any -- cgit v1.2.1 From b67db6b53f80f895237f58c1c8fa8a855cdd8051 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 13:50:50 +0200 Subject: ENH: Replace module-level `__getattr__` with explicitly defined objects --- numpy/fft/__init__.pyi | 20 ++++++++++++++++++-- numpy/linalg/__init__.pyi | 23 +++++++++++++++++++++-- numpy/polynomial/__init__.pyi | 9 +++++++-- numpy/version.pyi | 7 +++++-- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/numpy/fft/__init__.pyi b/numpy/fft/__init__.pyi index 3938d68de..45190517f 100644 --- a/numpy/fft/__init__.pyi +++ b/numpy/fft/__init__.pyi @@ -1,4 +1,20 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +fft: Any +ifft: Any +rfft: Any +irfft: Any +hfft: Any +ihfft: Any +rfftn: Any +irfftn: Any +rfft2: Any +irfft2: Any +fft2: Any +ifft2: Any +fftn: Any +ifftn: Any +fftshift: Any +ifftshift: Any +fftfreq: Any +rfftfreq: Any diff --git a/numpy/linalg/__init__.pyi b/numpy/linalg/__init__.pyi index 3938d68de..ffb05bb81 100644 --- a/numpy/linalg/__init__.pyi +++ b/numpy/linalg/__init__.pyi @@ -1,4 +1,23 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +matrix_power: Any +solve: Any +tensorsolve: Any +tensorinv: Any +inv: Any +cholesky: Any +eigvals: Any +eigvalsh: Any +pinv: Any +slogdet: Any +det: Any +svd: Any +eig: Any +eigh: Any +lstsq: Any +norm: Any +qr: Any +cond: Any +matrix_rank: Any +LinAlgError: Any +multi_dot: Any diff --git a/numpy/polynomial/__init__.pyi b/numpy/polynomial/__init__.pyi index 3938d68de..817ba22ac 100644 --- a/numpy/polynomial/__init__.pyi +++ b/numpy/polynomial/__init__.pyi @@ -1,4 +1,9 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +Polynomial: Any +Chebyshev: Any +Legendre: Any +Hermite: Any +HermiteE: Any +Laguerre: Any +set_default_printstyle: Any diff --git a/numpy/version.pyi b/numpy/version.pyi index 3938d68de..6f3659e43 100644 --- a/numpy/version.pyi +++ b/numpy/version.pyi @@ -1,4 +1,7 @@ from typing import Any -# TODO: remove when the full numpy namespace is defined -def __getattr__(name: str) -> Any: ... +short_version: Any +version: Any +full_version: Any +git_revision: Any +release: Any -- cgit v1.2.1 From 1a3efc43575434509fb968cbe42701094d9a0dee Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 13:53:08 +0200 Subject: ENH: Added placeholder stubs for `distutils` and `f2py` --- numpy/distutils/__init__.pyi | 4 ++++ numpy/f2py/__init__.pyi | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 numpy/distutils/__init__.pyi create mode 100644 numpy/f2py/__init__.pyi diff --git a/numpy/distutils/__init__.pyi b/numpy/distutils/__init__.pyi new file mode 100644 index 000000000..3938d68de --- /dev/null +++ b/numpy/distutils/__init__.pyi @@ -0,0 +1,4 @@ +from typing import Any + +# TODO: remove when the full numpy namespace is defined +def __getattr__(name: str) -> Any: ... diff --git a/numpy/f2py/__init__.pyi b/numpy/f2py/__init__.pyi new file mode 100644 index 000000000..602517957 --- /dev/null +++ b/numpy/f2py/__init__.pyi @@ -0,0 +1,5 @@ +from typing import Any + +run_main: Any +compile: Any +f2py_testing: Any -- cgit v1.2.1 From caa70eac1a5118f0de861f6898c52ac2e91dc803 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Thu, 20 Aug 2020 13:06:59 +0100 Subject: API: Remove `np.ctypeslib.ctypes_load_library` This function has been deprecated since fcee1ad856089a7ecb7b6865d280c0273dacb638 (Numpy v1.0b3). 14 years is more than enough time for users to switch from it. --- doc/release/upcoming_changes/17116.expired.rst | 2 ++ doc/source/reference/routines.ctypeslib.rst | 1 - numpy/ctypeslib.py | 8 ++------ 3 files changed, 4 insertions(+), 7 deletions(-) create mode 100644 doc/release/upcoming_changes/17116.expired.rst diff --git a/doc/release/upcoming_changes/17116.expired.rst b/doc/release/upcoming_changes/17116.expired.rst new file mode 100644 index 000000000..d8a3a43d5 --- /dev/null +++ b/doc/release/upcoming_changes/17116.expired.rst @@ -0,0 +1,2 @@ +* The 14-year deprecation of ``np.ctypeslib.ctypes_load_library`` is expired. + Use :func:`~numpy.ctypeslib.load_library` instead, which is identical. diff --git a/doc/source/reference/routines.ctypeslib.rst b/doc/source/reference/routines.ctypeslib.rst index 562638e9c..3a059f5d9 100644 --- a/doc/source/reference/routines.ctypeslib.rst +++ b/doc/source/reference/routines.ctypeslib.rst @@ -9,6 +9,5 @@ C-Types Foreign Function Interface (:mod:`numpy.ctypeslib`) .. autofunction:: as_array .. autofunction:: as_ctypes .. autofunction:: as_ctypes_type -.. autofunction:: ctypes_load_library .. autofunction:: load_library .. autofunction:: ndpointer diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index 76ba838b7..e8f7750fe 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -49,12 +49,11 @@ Then, we're ready to call ``foo_func``: >>> _lib.foo_func(out, len(out)) #doctest: +SKIP """ -__all__ = ['load_library', 'ndpointer', 'ctypes_load_library', - 'c_intp', 'as_ctypes', 'as_array'] +__all__ = ['load_library', 'ndpointer', 'c_intp', 'as_ctypes', 'as_array'] import os from numpy import ( - integer, ndarray, dtype as _dtype, deprecate, array, frombuffer + integer, ndarray, dtype as _dtype, array, frombuffer ) from numpy.core.multiarray import _flagdict, flagsobj @@ -75,7 +74,6 @@ if ctypes is None: """ raise ImportError("ctypes is not available.") - ctypes_load_library = _dummy load_library = _dummy as_ctypes = _dummy as_array = _dummy @@ -154,8 +152,6 @@ else: ## if no successful return in the libname_ext loop: raise OSError("no file with expected extension") - ctypes_load_library = deprecate(load_library, 'ctypes_load_library', - 'load_library') def _num_fromflags(flaglist): num = 0 -- cgit v1.2.1 From ced78172afefd21e1540765ebeaad1dada865380 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 14:14:14 +0200 Subject: REV: Removed deprecated `ctypeslib` stubs --- numpy/ctypeslib.pyi | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 numpy/ctypeslib.pyi diff --git a/numpy/ctypeslib.pyi b/numpy/ctypeslib.pyi deleted file mode 100644 index c71d2dda2..000000000 --- a/numpy/ctypeslib.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Any - -load_library: Any -ndpointer: Any -ctypes_load_library: Any -c_intp: Any -as_ctypes: Any -as_array: Any -- cgit v1.2.1 From c8533fdf56a871edfb239d92bb608cc9222f96bd Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 14:15:46 +0200 Subject: Revert "REV: Removed deprecated `ctypeslib` stubs" This reverts commit ced78172afefd21e1540765ebeaad1dada865380. --- numpy/ctypeslib.pyi | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 numpy/ctypeslib.pyi diff --git a/numpy/ctypeslib.pyi b/numpy/ctypeslib.pyi new file mode 100644 index 000000000..c71d2dda2 --- /dev/null +++ b/numpy/ctypeslib.pyi @@ -0,0 +1,8 @@ +from typing import Any + +load_library: Any +ndpointer: Any +ctypes_load_library: Any +c_intp: Any +as_ctypes: Any +as_array: Any -- cgit v1.2.1 From 2ec7afa3f4fcc5cb8792ac9f63ef38f667506c5a Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 14:17:00 +0200 Subject: DEP: Remove `ctypes_load_library` annotations --- numpy/ctypeslib.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/numpy/ctypeslib.pyi b/numpy/ctypeslib.pyi index c71d2dda2..cacc97d68 100644 --- a/numpy/ctypeslib.pyi +++ b/numpy/ctypeslib.pyi @@ -2,7 +2,6 @@ from typing import Any load_library: Any ndpointer: Any -ctypes_load_library: Any c_intp: Any as_ctypes: Any as_array: Any -- cgit v1.2.1 From 1c7c0f081711654095023a984978cd812ac94fb6 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Thu, 20 Aug 2020 14:17:23 +0200 Subject: TST: Remove `compat` and `core` from the tests --- numpy/tests/typing/reveal/modules.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/numpy/tests/typing/reveal/modules.py b/numpy/tests/typing/reveal/modules.py index 16709496b..406463152 100644 --- a/numpy/tests/typing/reveal/modules.py +++ b/numpy/tests/typing/reveal/modules.py @@ -3,8 +3,6 @@ import numpy as np reveal_type(np) # E: ModuleType reveal_type(np.char) # E: ModuleType -reveal_type(np.compat) # E: ModuleType -reveal_type(np.core) # E: ModuleType reveal_type(np.ctypeslib) # E: ModuleType reveal_type(np.emath) # E: ModuleType reveal_type(np.fft) # E: ModuleType -- cgit v1.2.1 From 4983fc897126ffe429f9d87e7b83152c61e237f6 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Wed, 19 Aug 2020 12:11:08 -0700 Subject: DOC: make spacing consistent in NEP 41 bullet points Currently the uneven spacing is making some of the bullet points show up as quoted; see e.g. https://numpy.org/neps/nep-0041-improved-dtype-support.html#backward-compatibility --- doc/neps/nep-0041-improved-dtype-support.rst | 52 ++++++++++++++-------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/doc/neps/nep-0041-improved-dtype-support.rst b/doc/neps/nep-0041-improved-dtype-support.rst index 56ff5eac6..6dc4ea50c 100644 --- a/doc/neps/nep-0041-improved-dtype-support.rst +++ b/doc/neps/nep-0041-improved-dtype-support.rst @@ -514,22 +514,22 @@ are not yet fully clear, we anticipate, and accept the following changes: * **C-API**: - * In old versions of NumPy ``PyArray_DescrCheck`` is a macro which uses - ``type(dtype) is np.dtype``. When compiling against an old NumPy version, - the macro may have to be replaced with the corresponding - ``PyObject_IsInstance`` call. (If this is a problem, we could backport - fixing the macro) - - * The UFunc machinery changes will break *limited* parts of the current - implementation. Replacing e.g. the default ``TypeResolver`` is expected - to remain supported for a time, although optimized masked inner loop iteration - (which is not even used *within* NumPy) will no longer be supported. - - * All functions currently defined on the dtypes, such as - ``PyArray_Descr->f->nonzero``, will be defined and accessed differently. - This means that in the long run lowlevel access code will - have to be changed to use the new API. Such changes are expected to be - necessary in very few project. + * In old versions of NumPy ``PyArray_DescrCheck`` is a macro which uses + ``type(dtype) is np.dtype``. When compiling against an old NumPy version, + the macro may have to be replaced with the corresponding + ``PyObject_IsInstance`` call. (If this is a problem, we could backport + fixing the macro) + + * The UFunc machinery changes will break *limited* parts of the current + implementation. Replacing e.g. the default ``TypeResolver`` is expected + to remain supported for a time, although optimized masked inner loop iteration + (which is not even used *within* NumPy) will no longer be supported. + + * All functions currently defined on the dtypes, such as + ``PyArray_Descr->f->nonzero``, will be defined and accessed differently. + This means that in the long run lowlevel access code will + have to be changed to use the new API. Such changes are expected to be + necessary in very few project. * **dtype implementors (C-API)**: @@ -541,16 +541,16 @@ are not yet fully clear, we anticipate, and accept the following changes: At least in some code paths, a similar mechanism is already used. * The ``scalarkind`` slot and registration of scalar casting will be - removed/ignored without replacement. - It currently allows partial value-based casting. - The ``PyArray_ScalarKind`` function will continue to work for builtin types, - but will not be used internally and be deprecated. - - * Currently user dtypes are defined as instances of ``np.dtype``. - The creation works by the user providing a prototype instance. - NumPy will need to modify at least the type during registration. - This has no effect for either ``rational`` or ``quaternion`` and mutation - of the structure seems unlikely after registration. + removed/ignored without replacement. + It currently allows partial value-based casting. + The ``PyArray_ScalarKind`` function will continue to work for builtin types, + but will not be used internally and be deprecated. + + * Currently user dtypes are defined as instances of ``np.dtype``. + The creation works by the user providing a prototype instance. + NumPy will need to modify at least the type during registration. + This has no effect for either ``rational`` or ``quaternion`` and mutation + of the structure seems unlikely after registration. Since there is a fairly large API surface concerning datatypes, further changes or the limitation certain function to currently existing datatypes is -- cgit v1.2.1 From 8e0b47524dd93cca23ad730304bb80c19cd65f15 Mon Sep 17 00:00:00 2001 From: Josh Wilson Date: Thu, 20 Aug 2020 12:19:43 -0700 Subject: ENH: allow running mypy through runtests.py Currently running mypy with the NumPy stubs configured requires either: - Installing NumPy - Adding the source directory to MYPYPATH and linking to the mypy.ini Both options are somewhat inconvenient, so add a `--mypy` option to runtests that handles setting things up for you. This will also be useful in the future for any typing codegen since it will ensure the project is built before type checking. --- runtests.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/runtests.py b/runtests.py index 8aefab0db..ce351e3c4 100755 --- a/runtests.py +++ b/runtests.py @@ -108,6 +108,8 @@ def main(argv): help="Start IPython shell with PYTHONPATH set") parser.add_argument("--shell", action="store_true", help="Start Unix shell with PYTHONPATH set") + parser.add_argument("--mypy", action="store_true", + help="Run mypy on files with NumPy on the MYPYPATH") parser.add_argument("--debug", "-g", action="store_true", help="Debug build") parser.add_argument("--parallel", "-j", type=int, default=0, @@ -131,7 +133,7 @@ def main(argv): "COMMIT. Note that you need to commit your " "changes first!")) parser.add_argument("args", metavar="ARGS", default=[], nargs=REMAINDER, - help="Arguments to pass to Nose, asv, Python or shell") + help="Arguments to pass to pytest, asv, mypy, Python or shell") args = parser.parse_args(argv) if args.durations < 0: @@ -211,6 +213,35 @@ def main(argv): subprocess.call([shell] + extra_argv) sys.exit(0) + if args.mypy: + try: + import mypy.api + except ImportError: + raise RuntimeError( + "Mypy not found. Please install it by running " + "pip install -r test_requirements.txt from the repo root" + ) + + os.environ['MYPYPATH'] = site_dir + # By default mypy won't color the output since it isn't being + # invoked from a tty. + os.environ['MYPY_FORCE_COLOR'] = '1' + + config = os.path.join( + site_dir, + "numpy", + "tests", + "typing", + "mypy.ini", + ) + + report, errors, status = mypy.api.run( + ['--config-file', config] + args.args + ) + print(report, end='') + print(errors, end='', file=sys.stderr) + sys.exit(status) + if args.coverage: dst_dir = os.path.join(ROOT_DIR, 'build', 'coverage') fn = os.path.join(dst_dir, 'coverage_html.js') -- cgit v1.2.1 From 0b2f5ed3fdf04874e23b19d03fa2938af146c203 Mon Sep 17 00:00:00 2001 From: karan-dhir Date: Fri, 21 Aug 2020 19:29:32 +0530 Subject: MAINT: changed ValueError line 103 in file timer_comparison --- numpy/ma/timer_comparison.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/ma/timer_comparison.py b/numpy/ma/timer_comparison.py index 83bd7852e..f5855efcf 100644 --- a/numpy/ma/timer_comparison.py +++ b/numpy/ma/timer_comparison.py @@ -100,9 +100,9 @@ class ModuleTester: header=header, names=('x', 'y')) assert cond, msg - except ValueError: + except ValueError as e: msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y')) - raise ValueError(msg) + raise ValueError(msg) from e def assert_array_equal(self, x, y, err_msg=''): """ -- cgit v1.2.1 From 9cebb29964a3f8767b433d26aee85d969c85ede6 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Thu, 20 Aug 2020 20:56:18 +0300 Subject: BUG: core: fix ilp64 blas dot/vdot/... for strides > int32 max Fix overlooked int cast when HAVE_BLAS_ILP64 is defined. It was supposed to cast to CBLAS_INT, not int. Also add a regression test. Move blas_stride() to npy_cblas.h Replace npy_is_aligned by modulo; we're going to call BLAS so no need to micro-optimize integer division here. --- numpy/core/src/common/npy_cblas.h | 26 ++++++++++++++++++++++++++ numpy/core/src/multiarray/common.h | 24 ------------------------ numpy/core/tests/test_regression.py | 15 ++++++++++++++- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/numpy/core/src/common/npy_cblas.h b/numpy/core/src/common/npy_cblas.h index 97308238a..c0441e81e 100644 --- a/numpy/core/src/common/npy_cblas.h +++ b/numpy/core/src/common/npy_cblas.h @@ -59,6 +59,32 @@ enum CBLAS_SIDE {CblasLeft=141, CblasRight=142}; #undef BLASINT #undef BLASNAME + +/* + * Convert NumPy stride to BLAS stride. Returns 0 if conversion cannot be done + * (BLAS won't handle negative or zero strides the way we want). + */ +static NPY_INLINE CBLAS_INT +blas_stride(npy_intp stride, unsigned itemsize) +{ + /* + * Should probably check pointer alignment also, but this may cause + * problems if we require complex to be 16 byte aligned. + */ + if (stride > 0 && (stride % itemsize) == 0) { + stride /= itemsize; +#ifndef HAVE_BLAS_ILP64 + if (stride <= INT_MAX) { +#else + if (stride <= NPY_MAX_INT64) { +#endif + return stride; + } + } + return 0; +} + + #ifdef __cplusplus } #endif diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 793cefaf8..e0b143604 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -292,30 +292,6 @@ npy_memchr(char * haystack, char needle, return p; } -/* - * Convert NumPy stride to BLAS stride. Returns 0 if conversion cannot be done - * (BLAS won't handle negative or zero strides the way we want). - */ -static NPY_INLINE int -blas_stride(npy_intp stride, unsigned itemsize) -{ - /* - * Should probably check pointer alignment also, but this may cause - * problems if we require complex to be 16 byte aligned. - */ - if (stride > 0 && npy_is_aligned((void *)stride, itemsize)) { - stride /= itemsize; -#ifndef HAVE_BLAS_ILP64 - if (stride <= INT_MAX) { -#else - if (stride <= NPY_MAX_INT64) { -#endif - return stride; - } - } - return 0; -} - /* * Define a chunksize for CBLAS. CBLAS counts in integers. */ diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index f778d4d7c..a97198076 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -14,7 +14,7 @@ from numpy.testing import ( assert_raises_regex, assert_warns, suppress_warnings, _assert_valid_refcount, HAS_REFCOUNT, ) -from numpy.testing._private.utils import _no_tracing +from numpy.testing._private.utils import _no_tracing, requires_memory from numpy.compat import asbytes, asunicode, pickle try: @@ -2501,3 +2501,16 @@ class TestRegression: formats=[np.int64, np.int64])) descr = np.array((1, 1), dtype=dt).__array_interface__['descr'] assert descr == [('', '|V8')] # instead of [(b'', '|V8')] + + @pytest.mark.skipif(sys.maxsize < 2 ** 31 + 1, reason='overflows 32-bit python') + @requires_memory(free_bytes=9e9) + def test_dot_big_stride(self): + # gh-17111 + # blas stride = stride//itemsize > int32 max + int32_max = np.iinfo(np.int32).max + n = int32_max + 3 + a = np.empty([n], dtype=np.float32) + b = a[::n-1] + b[...] = 1 + assert b.strides[0] > int32_max * b.dtype.itemsize + assert np.dot(b, b) == 2.0 -- cgit v1.2.1 From 51bb2173537c533c59638a04afa442597b4d02c0 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Fri, 21 Aug 2020 23:11:43 +0300 Subject: MAINT: npy_cblas.h: redefine NPY_CBLAS_CHUNK in terms of CBLAS_INT_MAX --- numpy/core/src/common/npy_cblas.h | 19 ++++++++++++++----- numpy/core/src/multiarray/common.h | 13 ------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/numpy/core/src/common/npy_cblas.h b/numpy/core/src/common/npy_cblas.h index c0441e81e..072993ec2 100644 --- a/numpy/core/src/common/npy_cblas.h +++ b/numpy/core/src/common/npy_cblas.h @@ -47,8 +47,10 @@ enum CBLAS_SIDE {CblasLeft=141, CblasRight=142}; #ifdef HAVE_BLAS_ILP64 #define CBLAS_INT npy_int64 +#define CBLAS_INT_MAX NPY_MAX_INT64 #else #define CBLAS_INT int +#define CBLAS_INT_MAX INT_MAX #endif #define BLASNAME(name) CBLAS_FUNC(name) @@ -73,17 +75,24 @@ blas_stride(npy_intp stride, unsigned itemsize) */ if (stride > 0 && (stride % itemsize) == 0) { stride /= itemsize; -#ifndef HAVE_BLAS_ILP64 - if (stride <= INT_MAX) { -#else - if (stride <= NPY_MAX_INT64) { -#endif + if (stride <= CBLAS_INT_MAX) { return stride; } } return 0; } +/* + * Define a chunksize for CBLAS. + * + * The chunksize is the greatest power of two less than CBLAS_INT_MAX. + */ +#if NPY_MAX_INTP > CBLAS_INT_MAX +# define NPY_CBLAS_CHUNK (CBLAS_INT_MAX / 2 + 1) +#else +# define NPY_CBLAS_CHUNK NPY_MAX_INTP +#endif + #ifdef __cplusplus } diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index e0b143604..4410825fa 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -292,19 +292,6 @@ npy_memchr(char * haystack, char needle, return p; } -/* - * Define a chunksize for CBLAS. CBLAS counts in integers. - */ -#if NPY_MAX_INTP > INT_MAX -# ifndef HAVE_BLAS_ILP64 -# define NPY_CBLAS_CHUNK (INT_MAX / 2 + 1) -# else -# define NPY_CBLAS_CHUNK (NPY_MAX_INT64 / 2 + 1) -# endif -#else -# define NPY_CBLAS_CHUNK NPY_MAX_INTP -#endif - #include "ucsnarrow.h" /* -- cgit v1.2.1 From 00fa130e704951757a1d76d928e741b77da0b4a7 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Sat, 22 Aug 2020 16:27:04 -0400 Subject: DOC: NEP-42: Fix a few typos. --- doc/neps/nep-0042-new-dtypes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index 1f476114f..b37555892 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -267,7 +267,7 @@ information is currently provided and will be defined on the class: deprecated. This may be relaxed if a use-case arises. Additionally, existing methods (and C-side fields) will be provided. -However, the fields ``kind`` and and ``char`` will be set to ``\0`` +However, the fields ``kind`` and ``char`` will be set to ``\0`` (NULL character) on the C-side. While discouraged, except for NumPy builtin types, ``kind`` both will return the ``__qualname__`` of the object to ensure uniqueness for all DTypes. @@ -307,7 +307,7 @@ is the ``np.datetime64`` scalar. A potential DType such as ``Categorical`` will not be required to have a clear type associated with it. Instead, the ``type`` may be ``object`` and the -categoircal's values are arbitrary objects. +categorical's values are arbitrary objects. Unlike with well-defined scalars, this ``type`` cannot not be used for the dtype discovery necessary for coercion (compare section `DType Discovery during Array Coercion`_). @@ -659,7 +659,7 @@ should be "minutes". Common DType Operations ^^^^^^^^^^^^^^^^^^^^^^^ -Numpy currently provides functions like ``np.result_type`` and +NumPy currently provides functions like ``np.result_type`` and ``np.promote_types`` for determining common types. These differ in that ``np.result_type`` can take arrays and scalars as input and implements value based promotion [1]_. @@ -972,7 +972,7 @@ In general we could implement certain casts, such as ``int8`` to ``int24`` even if the user only provides an ``int16 -> int24`` cast. This proposal currently does not provide this functionality. However, it could be extended in the future to either find such casts dynamically, -or at least allow ``adjust_descriptors`` to return arbitray ``dtypes``. +or at least allow ``adjust_descriptors`` to return arbitrary ``dtypes``. If ``CastingImpl[Int8, Int24].adjust_descriptors((int8, int24))`` returns ``(int16, int24)``, the actual casting process could be extended to include the ``int8 -> int16`` cast. Unlike the above example, which is limited -- cgit v1.2.1 From 4da857ae94d9ee9ebeeb67e83858ad31bc3c1838 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sat, 22 Aug 2020 11:23:55 -0600 Subject: MAINT: Make arrayprint str and repr the ndarray defaults. This removes the old default routines in 'strfunc.c' that were never used and looked to have unicode/byte problems. The functions now self initialize when called, so the explicit initialization when the arrayprint module is loaded is no longer needed. --- numpy/core/arrayprint.py | 3 - numpy/core/src/multiarray/strfuncs.c | 186 ++++++++--------------------------- 2 files changed, 42 insertions(+), 147 deletions(-) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 5d9642ea8..ad1530419 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -1628,6 +1628,3 @@ def set_string_function(f, repr=True): return multiarray.set_string_function(_default_array_str, 0) else: return multiarray.set_string_function(f, repr) - -set_string_function(_default_array_str, False) -set_string_function(_default_array_repr, True) diff --git a/numpy/core/src/multiarray/strfuncs.c b/numpy/core/src/multiarray/strfuncs.c index 363cbdba2..d9d9b7c0a 100644 --- a/numpy/core/src/multiarray/strfuncs.c +++ b/numpy/core/src/multiarray/strfuncs.c @@ -3,14 +3,25 @@ #include #include - #include "npy_pycompat.h" - +#include "npy_import.h" #include "strfuncs.h" static PyObject *PyArray_StrFunction = NULL; static PyObject *PyArray_ReprFunction = NULL; + +static void +npy_PyErr_SetStringChained(PyObject *type, const char *message) +{ + PyObject *exc, *val, *tb; + + PyErr_Fetch(&exc, &val, &tb); + PyErr_SetString(type, message); + npy_PyErr_ChainExceptionsCause(exc, val, tb); +} + + /*NUMPY_API * Set the array print function to be a Python function. */ @@ -36,164 +47,52 @@ PyArray_SetStringFunction(PyObject *op, int repr) } -/* - * Extend string. On failure, returns NULL and leaves *strp alone. - * XXX we do this in multiple places; time for a string library? - */ -static char * -extend_str(char **strp, Py_ssize_t n, Py_ssize_t *maxp) -{ - char *str = *strp; - Py_ssize_t new_cap; - - if (n >= *maxp - 16) { - new_cap = *maxp * 2; - - if (new_cap <= *maxp) { /* overflow */ - return NULL; - } - str = PyArray_realloc(*strp, new_cap); - if (str != NULL) { - *strp = str; - *maxp = new_cap; - } - } - return str; -} - - -static int -dump_data(char **string, Py_ssize_t *n, Py_ssize_t *max_n, char *data, int nd, - npy_intp const *dimensions, npy_intp const *strides, PyArrayObject* self) -{ - PyObject *op = NULL, *sp = NULL; - char *ostring; - npy_intp i, N, ret = 0; - -#define CHECK_MEMORY do { \ - if (extend_str(string, *n, max_n) == NULL) { \ - ret = -1; \ - goto end; \ - } \ - } while (0) - - if (nd == 0) { - if ((op = PyArray_GETITEM(self, data)) == NULL) { - return -1; - } - sp = PyObject_Repr(op); - if (sp == NULL) { - ret = -1; - goto end; - } - ostring = PyString_AsString(sp); - N = PyString_Size(sp)*sizeof(char); - *n += N; - CHECK_MEMORY; - memmove(*string + (*n - N), ostring, N); - } - else { - CHECK_MEMORY; - (*string)[*n] = '['; - *n += 1; - for (i = 0; i < dimensions[0]; i++) { - if (dump_data(string, n, max_n, - data + (*strides)*i, - nd - 1, dimensions + 1, - strides + 1, self) < 0) { - return -1; - } - CHECK_MEMORY; - if (i < dimensions[0] - 1) { - (*string)[*n] = ','; - (*string)[*n+1] = ' '; - *n += 2; - } - } - CHECK_MEMORY; - (*string)[*n] = ']'; - *n += 1; - } - -#undef CHECK_MEMORY - -end: - Py_XDECREF(op); - Py_XDECREF(sp); - return ret; -} - - -static PyObject * -array_repr_builtin(PyArrayObject *self, int repr) -{ - PyObject *ret; - char *string; - /* max_n initial value is arbitrary, dump_data will extend it */ - Py_ssize_t n = 0, max_n = PyArray_NBYTES(self) * 4 + 7; - - if ((string = PyArray_malloc(max_n)) == NULL) { - return PyErr_NoMemory(); - } - - if (dump_data(&string, &n, &max_n, PyArray_DATA(self), - PyArray_NDIM(self), PyArray_DIMS(self), - PyArray_STRIDES(self), self) < 0) { - PyArray_free(string); - return NULL; - } - - if (repr) { - if (PyArray_ISEXTENDED(self)) { - ret = PyUString_FromFormat("array(%s, '%c%d')", - string, - PyArray_DESCR(self)->type, - PyArray_DESCR(self)->elsize); - } - else { - ret = PyUString_FromFormat("array(%s, '%c')", - string, - PyArray_DESCR(self)->type); - } - } - else { - ret = PyUString_FromStringAndSize(string, n); - } - - PyArray_free(string); - return ret; -} - - NPY_NO_EXPORT PyObject * array_repr(PyArrayObject *self) { - PyObject *s; + static PyObject *repr = NULL; - if (PyArray_ReprFunction == NULL) { - s = array_repr_builtin(self, 1); + if (PyArray_ReprFunction != NULL) { + return PyObject_CallFunctionObjArgs(PyArray_ReprFunction, self, NULL); } - else { - s = PyObject_CallFunctionObjArgs(PyArray_ReprFunction, self, NULL); + + /* + * We need to do a delayed import here as initialization on module load + * leads to circular import problems. + */ + npy_cache_import("numpy.core.arrayprint", "_default_array_repr", &repr); + if (repr == NULL) { + npy_PyErr_SetStringChained(PyExc_RuntimeError, + "Unable to configure default ndarray.__repr__"); + return NULL; } - return s; + return PyObject_CallFunctionObjArgs(repr, self, NULL); } NPY_NO_EXPORT PyObject * array_str(PyArrayObject *self) { - PyObject *s; + static PyObject *str = NULL; - if (PyArray_StrFunction == NULL) { - s = array_repr_builtin(self, 0); + if (PyArray_StrFunction != NULL) { + return PyObject_CallFunctionObjArgs(PyArray_StrFunction, self, NULL); } - else { - s = PyObject_CallFunctionObjArgs(PyArray_StrFunction, self, NULL); + + /* + * We need to do a delayed import here as initialization on module load leads + * to circular import problems. + */ + npy_cache_import("numpy.core.arrayprint", "_default_array_str", &str); + if (str == NULL) { + npy_PyErr_SetStringChained(PyExc_RuntimeError, + "Unable to configure default ndarray.__str__"); + return NULL; } - return s; + return PyObject_CallFunctionObjArgs(str, self, NULL); } + NPY_NO_EXPORT PyObject * array_format(PyArrayObject *self, PyObject *args) { @@ -221,4 +120,3 @@ array_format(PyArrayObject *self, PyObject *args) ); } } - -- cgit v1.2.1 From adccacf32ea8c159b2397a6053f4b97543ec08a7 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Sat, 22 Aug 2020 19:48:35 -0400 Subject: MAINT: lib: Change handling of the expired financial functions. In a previous commit, the expired financial functions were removed from NumPy, and code was added to __init__.py using the module __getattr__ to raise a customized AttributeError on any attempt to reference the expired names in the numpy namespace. That change broke released versions astropy, which has code that imports the financial functions. astropy never calls the functions, so they never saw the deprecation warnings that have been in place since numpy 1.18. This means that attempting to use a released version of astropy with numpy 1.20 will cause astropy to crash with the custom AttributeError. In this commit, instead of raising an exception when one of the expired names is referenced, a warning is generated. If the function is *called*, an exception is raised. --- numpy/__init__.py | 15 ++++++++++----- numpy/lib/tests/test_financial_expired.py | 13 +++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index c594928ce..8dc805379 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -215,9 +215,8 @@ else: del Arrayterator # These names were removed in NumPy 1.20. For at least one release, - # attempts to access these names in the numpy namespace will have an - # error message that refers to NEP 32 and points to the numpy_financial - # library. + # attempts to access these names in the numpy namespace will trigger + # a warning, and calling the function will raise an exception. _financial_names = ['fv', 'ipmt', 'irr', 'mirr', 'nper', 'npv', 'pmt', 'ppmt', 'pv', 'rate'] __expired_attrs__ = { @@ -241,13 +240,19 @@ else: # module level getattr is only supported in 3.7 onwards # https://www.python.org/dev/peps/pep-0562/ def __getattr__(attr): - # Raise AttributeError for expired attributes + # Warn for expired attributes, and return a dummy function + # that always raises an exception. try: msg = __expired_attrs__[attr] except KeyError: pass else: - raise AttributeError(msg) + warnings.warn(msg, RuntimeWarning) + + def _expired(*args, **kwds): + raise RuntimeError(msg) + + return _expired # Emit warnings for deprecated attributes try: diff --git a/numpy/lib/tests/test_financial_expired.py b/numpy/lib/tests/test_financial_expired.py index e1d05da0c..66bb08026 100644 --- a/numpy/lib/tests/test_financial_expired.py +++ b/numpy/lib/tests/test_financial_expired.py @@ -3,10 +3,11 @@ import pytest import numpy as np +@pytest.mark.skipif(sys.version_info[:2] < (3, 7), + reason="requires python 3.7 or higher") def test_financial_expired(): - if sys.version_info[:2] >= (3, 7): - match = 'NEP 32' - else: - match = None - with pytest.raises(AttributeError, match=match): - np.fv + match = 'NEP 32' + with pytest.warns(RuntimeWarning, match=match): + func = np.fv + with pytest.raises(RuntimeError, match=match): + func(1, 2, 3) -- cgit v1.2.1 From dcb39277c201a418c653a1a98f9a6680fb3d5495 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 15:36:07 +0200 Subject: ENH: Add stubs for `np.core.function_base` --- numpy/__init__.pyi | 11 +++++++++ numpy/core/function_base.pyi | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 numpy/core/function_base.pyi diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index fad5e1774..60736e009 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -44,6 +44,17 @@ if sys.version_info >= (3, 8): else: from typing_extensions import Literal, Protocol +from numpy.core.function_base import ( + linspace, + logspace, + geomspace, +) + +# Add an object to `__all__` if their stubs are defined in an external file; +# their stubs will not be recognized otherwise. +# NOTE: This is redundant for objects defined within this file. +__all__ = ["linspace", "logspace", "geomspace"] + # TODO: remove when the full numpy namespace is defined def __getattr__(name: str) -> Any: ... diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi new file mode 100644 index 000000000..2edcb88c3 --- /dev/null +++ b/numpy/core/function_base.pyi @@ -0,0 +1,53 @@ +import sys +from typing import overload, Tuple, Union, Sequence, Any + +from numpy import ndarray, floating, number, _NumberLike +from numpy.typing import ArrayLike, DtypeLike, _SupportsArray + +if sys.version_info >= (3, 8): + from typing import SupportsIndex, Literal +else: + from typing_extensions import SupportsIndex, Literal + +# TODO: wait for support for recursive types +_ArrayLikeNested = Sequence[Sequence[Any]] +_ArrayLikeNumber = Union[ + _NumberLike, Sequence[_NumberLike], ndarray, _SupportsArray, _ArrayLikeNested +] +@overload +def linspace( + start: _ArrayLikeNumber, + stop: _ArrayLikeNumber, + num: SupportsIndex = ..., + endpoint: bool = ..., + retstep: Literal[False] = ..., + dtype: DtypeLike = ..., + axis: int = ..., +) -> ndarray: ... +@overload +def linspace( + start: _ArrayLikeNumber, + stop: _ArrayLikeNumber, + num: SupportsIndex = ..., + endpoint: bool = ..., + retstep: Literal[True] = ..., + dtype: DtypeLike = ..., + axis: int = ..., +) -> Tuple[ndarray, floating]: ... +def logspace( + start: _ArrayLikeNumber, + stop: _ArrayLikeNumber, + num: SupportsIndex = ..., + endpoint: bool = ..., + base: _ArrayLikeNumber = ..., + dtype: DtypeLike = ..., + axis: int = ..., +) -> ndarray: ... +def geomspace( + start: _ArrayLikeNumber, + stop: _ArrayLikeNumber, + num: SupportsIndex = ..., + endpoint: bool = ..., + dtype: DtypeLike = ..., + axis: int = ..., +) -> ndarray: ... -- cgit v1.2.1 From 9677fad41873ee3b2bb6f2130c9cbd197903d0b1 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 15:36:26 +0200 Subject: TST: Add tests for the `np.core.function_base` stubs --- numpy/tests/typing/fail/linspace.py | 13 +++++++++++++ numpy/tests/typing/pass/linspace.py | 17 +++++++++++++++++ numpy/tests/typing/reveal/linspace.py | 6 ++++++ 3 files changed, 36 insertions(+) create mode 100644 numpy/tests/typing/fail/linspace.py create mode 100644 numpy/tests/typing/pass/linspace.py create mode 100644 numpy/tests/typing/reveal/linspace.py diff --git a/numpy/tests/typing/fail/linspace.py b/numpy/tests/typing/fail/linspace.py new file mode 100644 index 000000000..a9769c5d6 --- /dev/null +++ b/numpy/tests/typing/fail/linspace.py @@ -0,0 +1,13 @@ +import numpy as np + +np.linspace(None, 'bob') # E: No overload variant +np.linspace(0, 2, num=10.0) # E: No overload variant +np.linspace(0, 2, endpoint='True') # E: No overload variant +np.linspace(0, 2, retstep=b'False') # E: No overload variant +np.linspace(0, 2, dtype=0) # E: No overload variant +np.linspace(0, 2, axis=None) # E: No overload variant + +np.logspace(None, 'bob') # E: Argument 1 +np.logspace(0, 2, base=None) # E: Argument "base" + +np.geomspace(None, 'bob') # E: Argument 1 diff --git a/numpy/tests/typing/pass/linspace.py b/numpy/tests/typing/pass/linspace.py new file mode 100644 index 000000000..ce427e866 --- /dev/null +++ b/numpy/tests/typing/pass/linspace.py @@ -0,0 +1,17 @@ +import numpy as np + +np.linspace(0, 2) +np.linspace(0.5, [0, 1, 2]) +np.linspace([0, 1, 2], 3) +np.linspace(0j, 2) +np.linspace(0, 2, num=10) +np.linspace(0, 2, endpoint=True) +np.linspace(0, 2, retstep=True) +np.linspace(0, 2, dtype=bool) +np.linspace([0, 1], [2, 3], axis=1) + +np.logspace(0, 2, base=2) +np.logspace(0, 2, base=2) +np.logspace(0, 2, base=[1j, 2j], num=2) + +np.geomspace(1, 2) diff --git a/numpy/tests/typing/reveal/linspace.py b/numpy/tests/typing/reveal/linspace.py new file mode 100644 index 000000000..678a5d3bf --- /dev/null +++ b/numpy/tests/typing/reveal/linspace.py @@ -0,0 +1,6 @@ +import numpy as np + +reveal_type(np.linspace(0, 10)) # E: numpy.ndarray +reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.floating] +reveal_type(np.logspace(0, 10)) # E: numpy.ndarray +reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray -- cgit v1.2.1 From d7e2003def0923bfce0f5339a571cffe90736252 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 15:38:04 +0200 Subject: TST: Bump the minimun `typing_extensions` version to 3.7.4.2 This is the first version that includes `SupportsIndex` --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 95c06e576..48387a6fd 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -14,4 +14,4 @@ cffi # - Mypy doesn't currently work on Python 3.9 # - Python 3.6 doesn't work because it doesn't understand py.typed mypy==0.782; platform_python_implementation != "PyPy" and python_version > "3.6" -typing_extensions +typing_extensions>=3.7.4.2 -- cgit v1.2.1 From cf1845f9d328bf2f3ab1d29b1892ad6eb69255a2 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 15:41:11 +0200 Subject: DOC: Clarify that the `base` parameter in `logspace` can be array_like --- numpy/core/function_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index b2f17cfeb..8a1fee99b 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -165,7 +165,7 @@ def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, if axis != 0: y = _nx.moveaxis(y, 0, axis) - + if _nx.issubdtype(dtype, _nx.integer): _nx.floor(y, out=y) @@ -207,7 +207,7 @@ def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, endpoint : boolean, optional If true, `stop` is the last sample. Otherwise, it is not included. Default is True. - base : float, optional + base : array_like, optional The base of the log space. The step size between the elements in ``ln(samples) / ln(base)`` (or ``log_base(samples)``) is uniform. Default is 10.0. -- cgit v1.2.1 From a90863a7193b2ff47e835d7d0313acc21d8e3969 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 15:51:14 +0200 Subject: MAINT: Removed unused import --- numpy/core/function_base.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 2edcb88c3..54328c48b 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -1,7 +1,7 @@ import sys from typing import overload, Tuple, Union, Sequence, Any -from numpy import ndarray, floating, number, _NumberLike +from numpy import ndarray, floating, _NumberLike from numpy.typing import ArrayLike, DtypeLike, _SupportsArray if sys.version_info >= (3, 8): -- cgit v1.2.1 From d10245fff16395cd83344543a59d470d24d7c2fd Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Sun, 23 Aug 2020 11:19:46 -0400 Subject: Use a better name for the collection of expired functions. --- numpy/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index 8dc805379..41c3dc42d 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -219,7 +219,7 @@ else: # a warning, and calling the function will raise an exception. _financial_names = ['fv', 'ipmt', 'irr', 'mirr', 'nper', 'npv', 'pmt', 'ppmt', 'pv', 'rate'] - __expired_attrs__ = { + __expired_functions__ = { name: (f'In accordance with NEP 32, the function {name} was removed ' 'from NumPy version 1.20. A replacement for this function ' 'is available in the numpy_financial library: ' @@ -243,7 +243,7 @@ else: # Warn for expired attributes, and return a dummy function # that always raises an exception. try: - msg = __expired_attrs__[attr] + msg = __expired_functions__[attr] except KeyError: pass else: -- cgit v1.2.1 From 72e11d38e3d8acadd1d5341e1c0b4b17d0eda6cb Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 17:50:55 +0200 Subject: REV: Manually define the `SupportsIndex` protocol `SupportsIndex` is apparently still missing from the typing-extensions 3.7.4.2 pypi release (see https://github.com/python/typing/issues/749): manually define the protocol in the relevant stubs. --- numpy/core/function_base.pyi | 5 ++++- test_requirements.txt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 54328c48b..973312c1c 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -7,7 +7,10 @@ from numpy.typing import ArrayLike, DtypeLike, _SupportsArray if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal else: - from typing_extensions import SupportsIndex, Literal + from typing_extensions import Literal, Protocol + + class SupportsIndex(Protocol): + def __index__(self) -> int: ... # TODO: wait for support for recursive types _ArrayLikeNested = Sequence[Sequence[Any]] diff --git a/test_requirements.txt b/test_requirements.txt index 48387a6fd..95c06e576 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -14,4 +14,4 @@ cffi # - Mypy doesn't currently work on Python 3.9 # - Python 3.6 doesn't work because it doesn't understand py.typed mypy==0.782; platform_python_implementation != "PyPy" and python_version > "3.6" -typing_extensions>=3.7.4.2 +typing_extensions -- cgit v1.2.1 From cabee5ecffe634d8f9aebe7f30427cbb1052aeb7 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 17:59:35 +0200 Subject: MAINT: The returned step size can be complex Co-Authored-By: Eric Wieser --- numpy/core/function_base.pyi | 4 ++-- numpy/tests/typing/pass/linspace.py | 1 + numpy/tests/typing/reveal/linspace.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 973312c1c..755fe8723 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -1,7 +1,7 @@ import sys from typing import overload, Tuple, Union, Sequence, Any -from numpy import ndarray, floating, _NumberLike +from numpy import ndarray, inexact, _NumberLike from numpy.typing import ArrayLike, DtypeLike, _SupportsArray if sys.version_info >= (3, 8): @@ -36,7 +36,7 @@ def linspace( retstep: Literal[True] = ..., dtype: DtypeLike = ..., axis: int = ..., -) -> Tuple[ndarray, floating]: ... +) -> Tuple[ndarray, inexact]: ... def logspace( start: _ArrayLikeNumber, stop: _ArrayLikeNumber, diff --git a/numpy/tests/typing/pass/linspace.py b/numpy/tests/typing/pass/linspace.py index ce427e866..c75f5bf66 100644 --- a/numpy/tests/typing/pass/linspace.py +++ b/numpy/tests/typing/pass/linspace.py @@ -7,6 +7,7 @@ np.linspace(0j, 2) np.linspace(0, 2, num=10) np.linspace(0, 2, endpoint=True) np.linspace(0, 2, retstep=True) +np.linspace(0j, 2j, retstep=True) np.linspace(0, 2, dtype=bool) np.linspace([0, 1], [2, 3], axis=1) diff --git a/numpy/tests/typing/reveal/linspace.py b/numpy/tests/typing/reveal/linspace.py index 678a5d3bf..cfbbdf390 100644 --- a/numpy/tests/typing/reveal/linspace.py +++ b/numpy/tests/typing/reveal/linspace.py @@ -1,6 +1,6 @@ import numpy as np reveal_type(np.linspace(0, 10)) # E: numpy.ndarray -reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.floating] +reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact] reveal_type(np.logspace(0, 10)) # E: numpy.ndarray reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray -- cgit v1.2.1 From c9c44fd65046fc9e5c7308e04522a9ff8562f649 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 18:53:44 +0200 Subject: MAINT: `SupportsIndex` was added to typing-extensions as of 3.7.4.3 --- numpy/core/function_base.pyi | 5 +---- test_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 755fe8723..5891d640e 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -7,10 +7,7 @@ from numpy.typing import ArrayLike, DtypeLike, _SupportsArray if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal else: - from typing_extensions import Literal, Protocol - - class SupportsIndex(Protocol): - def __index__(self) -> int: ... + from typing_extensions import SupportsIndex, Literal # TODO: wait for support for recursive types _ArrayLikeNested = Sequence[Sequence[Any]] diff --git a/test_requirements.txt b/test_requirements.txt index 95c06e576..49ecfe403 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -14,4 +14,4 @@ cffi # - Mypy doesn't currently work on Python 3.9 # - Python 3.6 doesn't work because it doesn't understand py.typed mypy==0.782; platform_python_implementation != "PyPy" and python_version > "3.6" -typing_extensions +typing_extensions>=3.7.4.3 -- cgit v1.2.1 From 68832b368d5fd69770ee09359ea8a6a4ae4aa312 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 20:43:26 +0200 Subject: Revert "MAINT: `SupportsIndex` was added to typing-extensions as of 3.7.4.3" This reverts commit c9c44fd65046fc9e5c7308e04522a9ff8562f649. --- numpy/core/function_base.pyi | 5 ++++- test_requirements.txt | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 5891d640e..755fe8723 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -7,7 +7,10 @@ from numpy.typing import ArrayLike, DtypeLike, _SupportsArray if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal else: - from typing_extensions import SupportsIndex, Literal + from typing_extensions import Literal, Protocol + + class SupportsIndex(Protocol): + def __index__(self) -> int: ... # TODO: wait for support for recursive types _ArrayLikeNested = Sequence[Sequence[Any]] diff --git a/test_requirements.txt b/test_requirements.txt index 49ecfe403..95c06e576 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -14,4 +14,4 @@ cffi # - Mypy doesn't currently work on Python 3.9 # - Python 3.6 doesn't work because it doesn't understand py.typed mypy==0.782; platform_python_implementation != "PyPy" and python_version > "3.6" -typing_extensions>=3.7.4.3 +typing_extensions -- cgit v1.2.1 From 423004b6d39d04028ed0b3397a4dd14681451eae Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Sun, 23 Aug 2020 23:08:43 +0200 Subject: ENH: Change the `axis` type to `SupportsIndex` --- numpy/core/function_base.pyi | 8 ++++---- numpy/tests/typing/pass/linspace.py | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 755fe8723..c6ebbd5f5 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -25,7 +25,7 @@ def linspace( endpoint: bool = ..., retstep: Literal[False] = ..., dtype: DtypeLike = ..., - axis: int = ..., + axis: SupportsIndex = ..., ) -> ndarray: ... @overload def linspace( @@ -35,7 +35,7 @@ def linspace( endpoint: bool = ..., retstep: Literal[True] = ..., dtype: DtypeLike = ..., - axis: int = ..., + axis: SupportsIndex = ..., ) -> Tuple[ndarray, inexact]: ... def logspace( start: _ArrayLikeNumber, @@ -44,7 +44,7 @@ def logspace( endpoint: bool = ..., base: _ArrayLikeNumber = ..., dtype: DtypeLike = ..., - axis: int = ..., + axis: SupportsIndex = ..., ) -> ndarray: ... def geomspace( start: _ArrayLikeNumber, @@ -52,5 +52,5 @@ def geomspace( num: SupportsIndex = ..., endpoint: bool = ..., dtype: DtypeLike = ..., - axis: int = ..., + axis: SupportsIndex = ..., ) -> ndarray: ... diff --git a/numpy/tests/typing/pass/linspace.py b/numpy/tests/typing/pass/linspace.py index c75f5bf66..8c6d0d56b 100644 --- a/numpy/tests/typing/pass/linspace.py +++ b/numpy/tests/typing/pass/linspace.py @@ -1,5 +1,9 @@ import numpy as np +class Index: + def __index__(self) -> int: + return 0 + np.linspace(0, 2) np.linspace(0.5, [0, 1, 2]) np.linspace([0, 1, 2], 3) @@ -9,7 +13,7 @@ np.linspace(0, 2, endpoint=True) np.linspace(0, 2, retstep=True) np.linspace(0j, 2j, retstep=True) np.linspace(0, 2, dtype=bool) -np.linspace([0, 1], [2, 3], axis=1) +np.linspace([0, 1], [2, 3], axis=Index()) np.logspace(0, 2, base=2) np.logspace(0, 2, base=2) -- cgit v1.2.1 From 152435d619d082beaa6e7735e7384f646bdc3000 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 23 Aug 2020 16:28:05 -0600 Subject: MAINT: Replace PyUString_* by PyUnicode_* equivalents. This is a straight substitution of the current defines in npy_3kcompat.h and should not cause any problems. The substitutions are * PyUString_Format -> PyUnicode_Format * PyUString_FromFormat -> PyUnicode_FromFormat * PyUString_FromString -> PyUnicode_FromString * PyUString_FromStringAndSize -> PyUnicode_FromStringAndSize * PyUString_InternFromString -> PyUnicode_InternFromString Because the names maintain the same length, no extra formatting was needed. --- numpy/core/src/common/array_assign.c | 4 +- numpy/core/src/multiarray/_multiarray_tests.c.src | 2 +- numpy/core/src/multiarray/arrayobject.c | 2 +- numpy/core/src/multiarray/arraytypes.c.src | 2 +- numpy/core/src/multiarray/buffer.c | 2 +- numpy/core/src/multiarray/common.c | 16 +++--- numpy/core/src/multiarray/convert.c | 4 +- numpy/core/src/multiarray/datetime.c | 36 +++++++------- numpy/core/src/multiarray/descriptor.c | 18 +++---- numpy/core/src/multiarray/dragon4.c | 4 +- numpy/core/src/multiarray/flagsobject.c | 2 +- numpy/core/src/multiarray/mapping.c | 8 +-- numpy/core/src/multiarray/multiarraymodule.c | 26 +++++----- numpy/core/src/multiarray/nditer_constr.c | 16 +++--- numpy/core/src/multiarray/nditer_pywrap.c | 2 +- numpy/core/src/multiarray/scalartypes.c.src | 60 +++++++++++------------ numpy/core/src/multiarray/shape.c | 12 ++--- numpy/core/src/umath/_rational_tests.c.src | 10 ++-- numpy/core/src/umath/extobj.c | 2 +- numpy/core/src/umath/override.c | 2 +- numpy/core/src/umath/ufunc_object.c | 10 ++-- numpy/core/src/umath/ufunc_type_resolution.c | 10 ++-- numpy/core/src/umath/umathmodule.c | 34 ++++++------- 23 files changed, 142 insertions(+), 142 deletions(-) diff --git a/numpy/core/src/common/array_assign.c b/numpy/core/src/common/array_assign.c index d626d1260..e365b49e4 100644 --- a/numpy/core/src/common/array_assign.c +++ b/numpy/core/src/common/array_assign.c @@ -67,12 +67,12 @@ broadcast_strides(int ndim, npy_intp const *shape, broadcast_error: { PyObject *errmsg; - errmsg = PyUString_FromFormat("could not broadcast %s from shape ", + errmsg = PyUnicode_FromFormat("could not broadcast %s from shape ", strides_name); PyUString_ConcatAndDel(&errmsg, build_shape_string(strides_ndim, strides_shape)); PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" into shape ")); + PyUnicode_FromString(" into shape ")); PyUString_ConcatAndDel(&errmsg, build_shape_string(ndim, shape)); PyErr_SetObject(PyExc_ValueError, errmsg); diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index da631c830..9c1fa0bad 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -1902,7 +1902,7 @@ PrintFloat_Printf_g(PyObject *obj, int precision) PyOS_snprintf(str, sizeof(str), "%.*g", precision, val); } - return PyUString_FromString(str); + return PyUnicode_FromString(str); } diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 95c650674..5da1b5f29 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -416,7 +416,7 @@ WARN_IN_DEALLOC(PyObject* warning, const char * msg) { if (PyErr_WarnEx(warning, msg, 1) < 0) { PyObject * s; - s = PyUString_FromString("array_dealloc"); + s = PyUnicode_FromString("array_dealloc"); if (s) { PyErr_WriteUnraisable(s); Py_DECREF(s); diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 9508fb5ad..3fee587b9 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -865,7 +865,7 @@ VOID_setitem(PyObject *op, void *input, void *vap) npy_intp names_size = PyTuple_GET_SIZE(descr->names); if (names_size != PyTuple_Size(op)) { - errmsg = PyUString_FromFormat( + errmsg = PyUnicode_FromFormat( "could not assign tuple of length %zd to structure " "with %" NPY_INTP_FMT " fields.", PyTuple_Size(op), names_size); diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 8b482dc03..5a1723e89 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -931,7 +931,7 @@ _descriptor_from_pep3118_format(char const *s) } *p = '\0'; - str = PyUString_FromStringAndSize(buf, strlen(buf)); + str = PyUnicode_FromStringAndSize(buf, strlen(buf)); if (str == NULL) { free(buf); return NULL; diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 49ef9e0e7..c131c456e 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -264,10 +264,10 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) for (i = 0; i < n && vals[i] < 0; i++); if (i == n) { - return PyUString_FromFormat("()%s", ending); + return PyUnicode_FromFormat("()%s", ending); } else { - ret = PyUString_FromFormat("(%" NPY_INTP_FMT, vals[i++]); + ret = PyUnicode_FromFormat("(%" NPY_INTP_FMT, vals[i++]); if (ret == NULL) { return NULL; } @@ -275,10 +275,10 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) for (; i < n; ++i) { if (vals[i] < 0) { - tmp = PyUString_FromString(",newaxis"); + tmp = PyUnicode_FromString(",newaxis"); } else { - tmp = PyUString_FromFormat(",%" NPY_INTP_FMT, vals[i]); + tmp = PyUnicode_FromFormat(",%" NPY_INTP_FMT, vals[i]); } if (tmp == NULL) { Py_DECREF(ret); @@ -292,10 +292,10 @@ convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending) } if (i == 1) { - tmp = PyUString_FromFormat(",)%s", ending); + tmp = PyUnicode_FromFormat(",)%s", ending); } else { - tmp = PyUString_FromFormat(")%s", ending); + tmp = PyUnicode_FromFormat(")%s", ending); } PyUString_ConcatAndDel(&ret, tmp); return ret; @@ -310,7 +310,7 @@ dot_alignment_error(PyArrayObject *a, int i, PyArrayObject *b, int j) *shape1 = NULL, *shape2 = NULL, *shape1_i = NULL, *shape2_j = NULL; - format = PyUString_FromString("shapes %s and %s not aligned:" + format = PyUnicode_FromString("shapes %s and %s not aligned:" " %d (dim %d) != %d (dim %d)"); shape1 = convert_shape_to_string(PyArray_NDIM(a), PyArray_DIMS(a), ""); @@ -333,7 +333,7 @@ dot_alignment_error(PyArrayObject *a, int i, PyArrayObject *b, int j) goto end; } - errmsg = PyUString_Format(format, fmt_args); + errmsg = PyUnicode_Format(format, fmt_args); if (errmsg != NULL) { PyErr_SetObject(PyExc_ValueError, errmsg); } diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index e7cbeaa77..af78bf18c 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -248,13 +248,13 @@ PyArray_ToFile(PyArrayObject *self, FILE *fp, char *sep, char *format) return -1; } PyTuple_SET_ITEM(tupobj,0,obj); - obj = PyUString_FromString((const char *)format); + obj = PyUnicode_FromString((const char *)format); if (obj == NULL) { Py_DECREF(tupobj); Py_DECREF(it); return -1; } - strobj = PyUString_Format(obj, tupobj); + strobj = PyUnicode_Format(obj, tupobj); Py_DECREF(obj); Py_DECREF(tupobj); if (strobj == NULL) { diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 8f3948c23..1be867d1f 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1435,14 +1435,14 @@ raise_if_datetime64_metadata_cast_error(char *object_type, } else { PyObject *errmsg; - errmsg = PyUString_FromFormat("Cannot cast %s " + errmsg = PyUnicode_FromFormat("Cannot cast %s " "from metadata ", object_type); errmsg = append_metastr_to_string(src_meta, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" to ")); + PyUnicode_FromString(" to ")); errmsg = append_metastr_to_string(dst_meta, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromFormat(" according to the rule %s", + PyUnicode_FromFormat(" according to the rule %s", npy_casting_to_string(casting))); PyErr_SetObject(PyExc_TypeError, errmsg); Py_DECREF(errmsg); @@ -1467,14 +1467,14 @@ raise_if_timedelta64_metadata_cast_error(char *object_type, } else { PyObject *errmsg; - errmsg = PyUString_FromFormat("Cannot cast %s " + errmsg = PyUnicode_FromFormat("Cannot cast %s " "from metadata ", object_type); errmsg = append_metastr_to_string(src_meta, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" to ")); + PyUnicode_FromString(" to ")); errmsg = append_metastr_to_string(dst_meta, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromFormat(" according to the rule %s", + PyUnicode_FromFormat(" according to the rule %s", npy_casting_to_string(casting))); PyErr_SetObject(PyExc_TypeError, errmsg); Py_DECREF(errmsg); @@ -1601,15 +1601,15 @@ compute_datetime_metadata_greatest_common_divisor( incompatible_units: { PyObject *errmsg; - errmsg = PyUString_FromString("Cannot get " + errmsg = PyUnicode_FromString("Cannot get " "a common metadata divisor for " "NumPy datetime metadata "); errmsg = append_metastr_to_string(meta1, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" and ")); + PyUnicode_FromString(" and ")); errmsg = append_metastr_to_string(meta2, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" because they have " + PyUnicode_FromString(" because they have " "incompatible nonlinear base time units")); PyErr_SetObject(PyExc_TypeError, errmsg); Py_DECREF(errmsg); @@ -1617,12 +1617,12 @@ incompatible_units: { } units_overflow: { PyObject *errmsg; - errmsg = PyUString_FromString("Integer overflow " + errmsg = PyUnicode_FromString("Integer overflow " "getting a common metadata divisor for " "NumPy datetime metadata "); errmsg = append_metastr_to_string(meta1, 0, errmsg); PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" and ")); + PyUnicode_FromString(" and ")); errmsg = append_metastr_to_string(meta2, 0, errmsg); PyErr_SetObject(PyExc_OverflowError, errmsg); Py_DECREF(errmsg); @@ -1747,7 +1747,7 @@ convert_datetime_metadata_to_tuple(PyArray_DatetimeMetaData *meta) } PyTuple_SET_ITEM(dt_tuple, 0, - PyUString_FromString(_datetime_strings[meta->base])); + PyUnicode_FromString(_datetime_strings[meta->base])); PyTuple_SET_ITEM(dt_tuple, 1, PyInt_FromLong(meta->num)); @@ -1771,7 +1771,7 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, if (!PyTuple_Check(tuple)) { PyObject *errmsg; - errmsg = PyUString_FromString("Require tuple for tuple to NumPy " + errmsg = PyUnicode_FromString("Require tuple for tuple to NumPy " "datetime metadata conversion, not "); PyUString_ConcatAndDel(&errmsg, PyObject_Repr(tuple)); PyErr_SetObject(PyExc_TypeError, errmsg); @@ -1973,7 +1973,7 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, if (meta->base == NPY_FR_GENERIC) { /* Without brackets, give a string "generic" */ if (skip_brackets) { - PyUString_ConcatAndDel(&ret, PyUString_FromString("generic")); + PyUString_ConcatAndDel(&ret, PyUnicode_FromString("generic")); return ret; } /* But with brackets, append nothing */ @@ -1994,18 +1994,18 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, if (num == 1) { if (skip_brackets) { - res = PyUString_FromFormat("%s", basestr); + res = PyUnicode_FromFormat("%s", basestr); } else { - res = PyUString_FromFormat("[%s]", basestr); + res = PyUnicode_FromFormat("[%s]", basestr); } } else { if (skip_brackets) { - res = PyUString_FromFormat("%d%s", num, basestr); + res = PyUnicode_FromFormat("%d%s", num, basestr); } else { - res = PyUString_FromFormat("[%d%s]", num, basestr); + res = PyUnicode_FromFormat("[%d%s]", num, basestr); } } diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 67d57975b..222ab471d 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -472,7 +472,7 @@ _convert_from_array_descr(PyObject *obj, int align) if (PyUnicode_GetLength(name) == 0) { Py_DECREF(name); if (title == NULL) { - name = PyUString_FromFormat("f%d", i); + name = PyUnicode_FromFormat("f%d", i); if (name == NULL) { goto fail; } @@ -673,7 +673,7 @@ _convert_from_list(PyObject *obj, int align) } PyTuple_SET_ITEM(tup, 0, (PyObject *)conv); PyTuple_SET_ITEM(tup, 1, size_obj); - PyObject *key = PyUString_FromFormat("f%d", i); + PyObject *key = PyUnicode_FromFormat("f%d", i); if (!key) { Py_DECREF(tup); goto fail; @@ -1887,10 +1887,10 @@ arraydescr_protocol_typestr_get(PyArray_Descr *self) size >>= 2; } if (self->type_num == NPY_OBJECT) { - ret = PyUString_FromFormat("%c%c", endian, basic_); + ret = PyUnicode_FromFormat("%c%c", endian, basic_); } else { - ret = PyUString_FromFormat("%c%c%d", endian, basic_, size); + ret = PyUnicode_FromFormat("%c%c%d", endian, basic_, size); } if (PyDataType_ISDATETIME(self)) { PyArray_DatetimeMetaData *meta; @@ -1974,7 +1974,7 @@ arraydescr_protocol_descr_get(PyArray_Descr *self) if (dobj == NULL) { return NULL; } - PyTuple_SET_ITEM(dobj, 0, PyUString_FromString("")); + PyTuple_SET_ITEM(dobj, 0, PyUnicode_FromString("")); PyTuple_SET_ITEM(dobj, 1, arraydescr_protocol_typestr_get(self)); res = PyList_New(1); if (res == NULL) { @@ -2450,7 +2450,7 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) if (self->type_num == NPY_UNICODE) { elsize >>= 2; } - obj = PyUString_FromFormat("%c%d",self->kind, elsize); + obj = PyUnicode_FromFormat("%c%d",self->kind, elsize); } PyTuple_SET_ITEM(ret, 1, Py_BuildValue("(NOO)", obj, Py_False, Py_True)); @@ -2492,7 +2492,7 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) PyTuple_SET_ITEM(state, 0, PyInt_FromLong(3)); } - PyTuple_SET_ITEM(state, 1, PyUString_FromFormat("%c", endian)); + PyTuple_SET_ITEM(state, 1, PyUnicode_FromFormat("%c", endian)); PyTuple_SET_ITEM(state, 2, arraydescr_subdescr_get(self)); if (PyDataType_HASFIELDS(self)) { Py_INCREF(self->names); @@ -2894,7 +2894,7 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) PyArray_DatetimeMetaData temp_dt_data; if ((! PyTuple_Check(metadata)) || (PyTuple_Size(metadata) != 2)) { - errmsg = PyUString_FromString("Invalid datetime dtype (metadata, c_metadata): "); + errmsg = PyUnicode_FromString("Invalid datetime dtype (metadata, c_metadata): "); PyUString_ConcatAndDel(&errmsg, PyObject_Repr(metadata)); PyErr_SetObject(PyExc_ValueError, errmsg); Py_DECREF(errmsg); @@ -3393,7 +3393,7 @@ arraydescr_field_subset_view(PyArray_Descr *self, PyObject *ind) /* disallow duplicate field indices */ if (PyDict_Contains(fields, name)) { PyObject *msg = NULL; - PyObject *fmt = PyUString_FromString( + PyObject *fmt = PyUnicode_FromString( "duplicate field of name {!r}"); if (fmt != NULL) { msg = PyObject_CallMethod(fmt, "format", "O", name); diff --git a/numpy/core/src/multiarray/dragon4.c b/numpy/core/src/multiarray/dragon4.c index 553d0effb..a7b252a77 100644 --- a/numpy/core/src/multiarray/dragon4.c +++ b/numpy/core/src/multiarray/dragon4.c @@ -3093,7 +3093,7 @@ Dragon4_Positional_##Type##_opt(npy_type *val, Dragon4_Options *opt)\ free_dragon4_bigint_scratch(scratch);\ return NULL;\ }\ - ret = PyUString_FromString(scratch->repr);\ + ret = PyUnicode_FromString(scratch->repr);\ free_dragon4_bigint_scratch(scratch);\ return ret;\ }\ @@ -3130,7 +3130,7 @@ Dragon4_Scientific_##Type##_opt(npy_type *val, Dragon4_Options *opt)\ free_dragon4_bigint_scratch(scratch);\ return NULL;\ }\ - ret = PyUString_FromString(scratch->repr);\ + ret = PyUnicode_FromString(scratch->repr);\ free_dragon4_bigint_scratch(scratch);\ return ret;\ }\ diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index d5f24e75a..b7d77e912 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -711,7 +711,7 @@ arrayflags_print(PyArrayFlagsObject *self) if (fl & NPY_ARRAY_WARN_ON_WRITE) { _warn_on_write = " (with WARN_ON_WRITE=True)"; } - return PyUString_FromFormat( + return PyUnicode_FromFormat( " %s : %s\n %s : %s\n" " %s : %s\n %s : %s%s\n" " %s : %s\n %s : %s\n" diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index db15ff1d5..c0cea0f21 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -1418,7 +1418,7 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view) return 0; } else if (tup == NULL){ - PyObject *errmsg = PyUString_FromString("no field of name "); + PyObject *errmsg = PyUnicode_FromString("no field of name "); PyUString_Concat(&errmsg, ind); PyErr_SetObject(PyExc_ValueError, errmsg); Py_DECREF(errmsg); @@ -2438,7 +2438,7 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, * Attempt to set a meaningful exception. Could also find out * if a boolean index was converted. */ - errmsg = PyUString_FromString("shape mismatch: indexing arrays could not " + errmsg = PyUnicode_FromString("shape mismatch: indexing arrays could not " "be broadcast together with shapes "); if (errmsg == NULL) { return -1; @@ -3183,7 +3183,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, goto finish; broadcast_error: - errmsg = PyUString_FromString("shape mismatch: value array " + errmsg = PyUnicode_FromString("shape mismatch: value array " "of shape "); if (errmsg == NULL) { goto finish; @@ -3204,7 +3204,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, goto finish; } - tmp = PyUString_FromString("could not be broadcast to indexing " + tmp = PyUnicode_FromString("could not be broadcast to indexing " "result of shape "); PyUString_ConcatAndDel(&errmsg, tmp); if (errmsg == NULL) { diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 7c5ceb962..a0310fdee 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4335,18 +4335,18 @@ NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_axis2 = NULL; static int intern_strings(void) { - npy_ma_str_array = PyUString_InternFromString("__array__"); - npy_ma_str_array_prepare = PyUString_InternFromString("__array_prepare__"); - npy_ma_str_array_wrap = PyUString_InternFromString("__array_wrap__"); - npy_ma_str_array_finalize = PyUString_InternFromString("__array_finalize__"); - npy_ma_str_ufunc = PyUString_InternFromString("__array_ufunc__"); - npy_ma_str_implementation = PyUString_InternFromString("_implementation"); - npy_ma_str_order = PyUString_InternFromString("order"); - npy_ma_str_copy = PyUString_InternFromString("copy"); - npy_ma_str_dtype = PyUString_InternFromString("dtype"); - npy_ma_str_ndmin = PyUString_InternFromString("ndmin"); - npy_ma_str_axis1 = PyUString_InternFromString("axis1"); - npy_ma_str_axis2 = PyUString_InternFromString("axis2"); + npy_ma_str_array = PyUnicode_InternFromString("__array__"); + npy_ma_str_array_prepare = PyUnicode_InternFromString("__array_prepare__"); + npy_ma_str_array_wrap = PyUnicode_InternFromString("__array_wrap__"); + npy_ma_str_array_finalize = PyUnicode_InternFromString("__array_finalize__"); + npy_ma_str_ufunc = PyUnicode_InternFromString("__array_ufunc__"); + npy_ma_str_implementation = PyUnicode_InternFromString("_implementation"); + npy_ma_str_order = PyUnicode_InternFromString("order"); + npy_ma_str_copy = PyUnicode_InternFromString("copy"); + npy_ma_str_dtype = PyUnicode_InternFromString("dtype"); + npy_ma_str_ndmin = PyUnicode_InternFromString("ndmin"); + npy_ma_str_axis1 = PyUnicode_InternFromString("axis1"); + npy_ma_str_axis2 = PyUnicode_InternFromString("axis2"); return npy_ma_str_array && npy_ma_str_array_prepare && npy_ma_str_array_wrap && npy_ma_str_array_finalize && @@ -4506,7 +4506,7 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { PyDict_SetItemString(d, "tracemalloc_domain", s); Py_DECREF(s); - s = PyUString_FromString("3.1"); + s = PyUnicode_FromString("3.1"); PyDict_SetItemString(d, "__version__", s); Py_DECREF(s); diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index a0dda4090..4bc6d2ca1 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -1755,7 +1755,7 @@ broadcast_error: { char *tmpstr; if (op_axes == NULL) { - errmsg = PyUString_FromString("operands could not be broadcast " + errmsg = PyUnicode_FromString("operands could not be broadcast " "together with shapes "); if (errmsg == NULL) { return 0; @@ -1776,7 +1776,7 @@ broadcast_error: { } } if (itershape != NULL) { - tmp = PyUString_FromString("and requested shape "); + tmp = PyUnicode_FromString("and requested shape "); if (tmp == NULL) { Py_DECREF(errmsg); return 0; @@ -1801,7 +1801,7 @@ broadcast_error: { Py_DECREF(errmsg); } else { - errmsg = PyUString_FromString("operands could not be broadcast " + errmsg = PyUnicode_FromString("operands could not be broadcast " "together with remapped shapes " "[original->remapped]: "); for (iop = 0; iop < nop; ++iop) { @@ -1843,7 +1843,7 @@ broadcast_error: { } } if (itershape != NULL) { - tmp = PyUString_FromString("and requested shape "); + tmp = PyUnicode_FromString("and requested shape "); if (tmp == NULL) { Py_DECREF(errmsg); return 0; @@ -1877,11 +1877,11 @@ operand_different_than_broadcast: { /* Start of error message */ if (op_flags[iop] & NPY_ITER_READONLY) { - errmsg = PyUString_FromString("non-broadcastable operand " + errmsg = PyUnicode_FromString("non-broadcastable operand " "with shape "); } else { - errmsg = PyUString_FromString("non-broadcastable output " + errmsg = PyUnicode_FromString("non-broadcastable output " "operand with shape "); } if (errmsg == NULL) { @@ -1913,7 +1913,7 @@ operand_different_than_broadcast: { } } - tmp = PyUString_FromString(" [remapped to "); + tmp = PyUnicode_FromString(" [remapped to "); if (tmp == NULL) { return 0; } @@ -1932,7 +1932,7 @@ operand_different_than_broadcast: { } } - tmp = PyUString_FromString(" doesn't match the broadcast shape "); + tmp = PyUnicode_FromString(" doesn't match the broadcast shape "); if (tmp == NULL) { return 0; } diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 1c68a4803..40cf72b67 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -1142,7 +1142,7 @@ npyiter_dealloc(NewNpyArrayIterObject *self) "results.", 1) < 0) { PyObject *s; - s = PyUString_FromString("npyiter_dealloc"); + s = PyUnicode_FromString("npyiter_dealloc"); if (s) { PyErr_WriteUnraisable(s); Py_DECREF(s); diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 58b9e2c30..715b7cbba 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -447,7 +447,7 @@ _void_to_hex(const char* argbuf, const Py_ssize_t arglen, } memcpy(&retbuf[j], echars, strlen(echars)); - retval = PyUString_FromStringAndSize(retbuf, slen); + retval = PyUnicode_FromStringAndSize(retbuf, slen); PyMem_Free(retbuf); return retval; @@ -518,21 +518,21 @@ datetimetype_repr(PyObject *self) */ if ((scal->obmeta.num == 1 && scal->obmeta.base != NPY_FR_h) || scal->obmeta.base == NPY_FR_GENERIC) { - ret = PyUString_FromString("numpy.datetime64('"); + ret = PyUnicode_FromString("numpy.datetime64('"); PyUString_ConcatAndDel(&ret, - PyUString_FromString(iso)); + PyUnicode_FromString(iso)); PyUString_ConcatAndDel(&ret, - PyUString_FromString("')")); + PyUnicode_FromString("')")); } else { - ret = PyUString_FromString("numpy.datetime64('"); + ret = PyUnicode_FromString("numpy.datetime64('"); PyUString_ConcatAndDel(&ret, - PyUString_FromString(iso)); + PyUnicode_FromString(iso)); PyUString_ConcatAndDel(&ret, - PyUString_FromString("','")); + PyUnicode_FromString("','")); ret = append_metastr_to_string(&scal->obmeta, 1, ret); PyUString_ConcatAndDel(&ret, - PyUString_FromString("')")); + PyUnicode_FromString("')")); } return ret; @@ -554,31 +554,31 @@ timedeltatype_repr(PyObject *self) /* The value */ if (scal->obval == NPY_DATETIME_NAT) { - ret = PyUString_FromString("numpy.timedelta64('NaT'"); + ret = PyUnicode_FromString("numpy.timedelta64('NaT'"); } else { /* * Can't use "%lld" if HAVE_LONG_LONG is not defined */ #if defined(HAVE_LONG_LONG) - ret = PyUString_FromFormat("numpy.timedelta64(%lld", + ret = PyUnicode_FromFormat("numpy.timedelta64(%lld", (long long)scal->obval); #else - ret = PyUString_FromFormat("numpy.timedelta64(%ld", + ret = PyUnicode_FromFormat("numpy.timedelta64(%ld", (long)scal->obval); #endif } /* The metadata unit */ if (scal->obmeta.base == NPY_FR_GENERIC) { PyUString_ConcatAndDel(&ret, - PyUString_FromString(")")); + PyUnicode_FromString(")")); } else { PyUString_ConcatAndDel(&ret, - PyUString_FromString(",'")); + PyUnicode_FromString(",'")); ret = append_metastr_to_string(&scal->obmeta, 1, ret); PyUString_ConcatAndDel(&ret, - PyUString_FromString("')")); + PyUnicode_FromString("')")); } return ret; @@ -611,7 +611,7 @@ datetimetype_str(PyObject *self) return NULL; } - return PyUString_FromString(iso); + return PyUnicode_FromString(iso); } static char *_datetime_verbose_strings[NPY_DATETIME_NUMUNITS] = { @@ -657,21 +657,21 @@ timedeltatype_str(PyObject *self) } if (scal->obval == NPY_DATETIME_NAT) { - ret = PyUString_FromString("NaT"); + ret = PyUnicode_FromString("NaT"); } else { /* * Can't use "%lld" if HAVE_LONG_LONG is not defined */ #if defined(HAVE_LONG_LONG) - ret = PyUString_FromFormat("%lld ", + ret = PyUnicode_FromFormat("%lld ", (long long)(scal->obval * scal->obmeta.num)); #else - ret = PyUString_FromFormat("%ld ", + ret = PyUnicode_FromFormat("%ld ", (long)(scal->obval * scal->obmeta.num)); #endif PyUString_ConcatAndDel(&ret, - PyUString_FromString(basestr)); + PyUnicode_FromString(basestr)); } return ret; @@ -795,7 +795,7 @@ legacy_@name@_format@kind@(@type@ val) PyOS_snprintf(buf, sizeof(buf), "(%s%sj)", re, im); } - return PyUString_FromString(buf); + return PyUnicode_FromString(buf); } #undef _FMT1 @@ -836,7 +836,7 @@ legacy_@name@_format@kind@(npy_@name@ val){ strcpy(&buf[cnt],".0"); } - return PyUString_FromString(buf); + return PyUnicode_FromString(buf); } #undef _FMT1 @@ -904,7 +904,7 @@ c@name@type_@kind@(PyObject *self) return NULL; } - PyUString_ConcatAndDel(&istr, PyUString_FromString("j")); + PyUString_ConcatAndDel(&istr, PyUnicode_FromString("j")); return istr; } @@ -915,13 +915,13 @@ c@name@type_@kind@(PyObject *self) } } else if (npy_isnan(val.real)) { - rstr = PyUString_FromString("nan"); + rstr = PyUnicode_FromString("nan"); } else if (val.real > 0){ - rstr = PyUString_FromString("inf"); + rstr = PyUnicode_FromString("inf"); } else { - rstr = PyUString_FromString("-inf"); + rstr = PyUnicode_FromString("-inf"); } if (npy_isfinite(val.imag)) { @@ -931,19 +931,19 @@ c@name@type_@kind@(PyObject *self) } } else if (npy_isnan(val.imag)) { - istr = PyUString_FromString("+nan"); + istr = PyUnicode_FromString("+nan"); } else if (val.imag > 0){ - istr = PyUString_FromString("+inf"); + istr = PyUnicode_FromString("+inf"); } else { - istr = PyUString_FromString("-inf"); + istr = PyUnicode_FromString("-inf"); } - ret = PyUString_FromString("("); + ret = PyUnicode_FromString("("); PyUString_ConcatAndDel(&ret, rstr); PyUString_ConcatAndDel(&ret, istr); - PyUString_ConcatAndDel(&ret, PyUString_FromString("j)")); + PyUString_ConcatAndDel(&ret, PyUnicode_FromString("j)")); return ret; } diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 30507112d..997b3bed2 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -458,7 +458,7 @@ _attempt_nocopy_reshape(PyArrayObject *self, int newnd, const npy_intp *newdims, static void raise_reshape_size_mismatch(PyArray_Dims *newshape, PyArrayObject *arr) { - PyObject *msg = PyUString_FromFormat("cannot reshape array of size %zd " + PyObject *msg = PyUnicode_FromFormat("cannot reshape array of size %zd " "into shape ", PyArray_SIZE(arr)); PyObject *tmp = convert_shape_to_string(newshape->len, newshape->ptr, ""); @@ -997,10 +997,10 @@ build_shape_string(npy_intp n, npy_intp const *vals) } if (i == n) { - return PyUString_FromFormat("()"); + return PyUnicode_FromFormat("()"); } else { - ret = PyUString_FromFormat("(%" NPY_INTP_FMT, vals[i++]); + ret = PyUnicode_FromFormat("(%" NPY_INTP_FMT, vals[i++]); if (ret == NULL) { return NULL; } @@ -1008,10 +1008,10 @@ build_shape_string(npy_intp n, npy_intp const *vals) for (; i < n; ++i) { if (vals[i] < 0) { - tmp = PyUString_FromString(",newaxis"); + tmp = PyUnicode_FromString(",newaxis"); } else { - tmp = PyUString_FromFormat(",%" NPY_INTP_FMT, vals[i]); + tmp = PyUnicode_FromFormat(",%" NPY_INTP_FMT, vals[i]); } if (tmp == NULL) { Py_DECREF(ret); @@ -1024,7 +1024,7 @@ build_shape_string(npy_intp n, npy_intp const *vals) } } - tmp = PyUString_FromFormat(")"); + tmp = PyUnicode_FromFormat(")"); PyUString_ConcatAndDel(&ret, tmp); return ret; } diff --git a/numpy/core/src/umath/_rational_tests.c.src b/numpy/core/src/umath/_rational_tests.c.src index 13e33d0a5..cbb6d9d17 100644 --- a/numpy/core/src/umath/_rational_tests.c.src +++ b/numpy/core/src/umath/_rational_tests.c.src @@ -526,11 +526,11 @@ static PyObject* pyrational_repr(PyObject* self) { rational x = ((PyRational*)self)->r; if (d(x)!=1) { - return PyUString_FromFormat( + return PyUnicode_FromFormat( "rational(%ld,%ld)",(long)x.n,(long)d(x)); } else { - return PyUString_FromFormat( + return PyUnicode_FromFormat( "rational(%ld)",(long)x.n); } } @@ -539,11 +539,11 @@ static PyObject* pyrational_str(PyObject* self) { rational x = ((PyRational*)self)->r; if (d(x)!=1) { - return PyUString_FromFormat( + return PyUnicode_FromFormat( "%ld/%ld",(long)x.n,(long)d(x)); } else { - return PyUString_FromFormat( + return PyUnicode_FromFormat( "%ld",(long)x.n); } } @@ -1126,7 +1126,7 @@ PyMODINIT_FUNC PyInit__rational_tests(void) { if (PyErr_Occurred()) { goto fail; } - numpy_str = PyUString_FromString("numpy"); + numpy_str = PyUnicode_FromString("numpy"); if (!numpy_str) { goto fail; } diff --git a/numpy/core/src/umath/extobj.c b/numpy/core/src/umath/extobj.c index 3404a0c6a..953d289c7 100644 --- a/numpy/core/src/umath/extobj.c +++ b/numpy/core/src/umath/extobj.c @@ -109,7 +109,7 @@ _error_handler(int method, PyObject *errobj, char *errtype, int retstatus, int * errtype, name); goto fail; } - args = Py_BuildValue("NN", PyUString_FromString(errtype), + args = Py_BuildValue("NN", PyUnicode_FromString(errtype), PyInt_FromLong((long) retstatus)); if (args == NULL) { goto fail; diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index bf6e5a698..a0090e302 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -605,7 +605,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, goto fail; } - method_name = PyUString_FromString(method); + method_name = PyUnicode_FromString(method); if (method_name == NULL) { goto fail; } diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index b47ccd291..4ed42fd1c 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -5383,7 +5383,7 @@ ufunc_dealloc(PyUFuncObject *ufunc) static PyObject * ufunc_repr(PyUFuncObject *ufunc) { - return PyUString_FromFormat("", ufunc->name); + return PyUnicode_FromFormat("", ufunc->name); } static int @@ -5995,7 +5995,7 @@ ufunc_get_doc(PyUFuncObject *ufunc) } if (ufunc->doc != NULL) { PyUString_ConcatAndDel(&doc, - PyUString_FromFormat("\n\n%s", ufunc->doc)); + PyUnicode_FromFormat("\n\n%s", ufunc->doc)); } return doc; } @@ -6051,7 +6051,7 @@ ufunc_get_types(PyUFuncObject *ufunc) t[ni + 2 + j] = _typecharfromnum(ufunc->types[n]); n++; } - str = PyUString_FromStringAndSize(t, no + ni + 2); + str = PyUnicode_FromStringAndSize(t, no + ni + 2); PyList_SET_ITEM(list, k, str); } PyArray_free(t); @@ -6061,7 +6061,7 @@ ufunc_get_types(PyUFuncObject *ufunc) static PyObject * ufunc_get_name(PyUFuncObject *ufunc) { - return PyUString_FromString(ufunc->name); + return PyUnicode_FromString(ufunc->name); } static PyObject * @@ -6077,7 +6077,7 @@ ufunc_get_signature(PyUFuncObject *ufunc) if (!ufunc->core_enabled) { Py_RETURN_NONE; } - return PyUString_FromString(ufunc->core_signature); + return PyUnicode_FromString(ufunc->core_signature); } #undef _typecharfromnum diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index ea20bb24f..42acacddf 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -36,15 +36,15 @@ npy_casting_to_py_object(NPY_CASTING casting) { switch (casting) { case NPY_NO_CASTING: - return PyUString_FromString("no"); + return PyUnicode_FromString("no"); case NPY_EQUIV_CASTING: - return PyUString_FromString("equiv"); + return PyUnicode_FromString("equiv"); case NPY_SAFE_CASTING: - return PyUString_FromString("safe"); + return PyUnicode_FromString("safe"); case NPY_SAME_KIND_CASTING: - return PyUString_FromString("same_kind"); + return PyUnicode_FromString("same_kind"); case NPY_UNSAFE_CASTING: - return PyUString_FromString("unsafe"); + return PyUnicode_FromString("unsafe"); default: return PyInt_FromLong(casting); } diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 708a27ad0..ba7ac1706 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -237,23 +237,23 @@ NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_pyvals_name = NULL; static int intern_strings(void) { - if (!(npy_um_str_out = PyUString_InternFromString("out"))) return -1; - if (!(npy_um_str_where = PyUString_InternFromString("where"))) return -1; - if (!(npy_um_str_axes = PyUString_InternFromString("axes"))) return -1; - if (!(npy_um_str_axis = PyUString_InternFromString("axis"))) return -1; - if (!(npy_um_str_keepdims = PyUString_InternFromString("keepdims"))) return -1; - if (!(npy_um_str_casting = PyUString_InternFromString("casting"))) return -1; - if (!(npy_um_str_order = PyUString_InternFromString("order"))) return -1; - if (!(npy_um_str_dtype = PyUString_InternFromString("dtype"))) return -1; - if (!(npy_um_str_subok = PyUString_InternFromString("subok"))) return -1; - if (!(npy_um_str_signature = PyUString_InternFromString("signature"))) return -1; - if (!(npy_um_str_sig = PyUString_InternFromString("sig"))) return -1; - if (!(npy_um_str_extobj = PyUString_InternFromString("extobj"))) return -1; - if (!(npy_um_str_array_prepare = PyUString_InternFromString("__array_prepare__"))) return -1; - if (!(npy_um_str_array_wrap = PyUString_InternFromString("__array_wrap__"))) return -1; - if (!(npy_um_str_array_finalize = PyUString_InternFromString("__array_finalize__"))) return -1; - if (!(npy_um_str_ufunc = PyUString_InternFromString("__array_ufunc__"))) return -1; - if (!(npy_um_str_pyvals_name = PyUString_InternFromString(UFUNC_PYVALS_NAME))) return -1; + if (!(npy_um_str_out = PyUnicode_InternFromString("out"))) return -1; + if (!(npy_um_str_where = PyUnicode_InternFromString("where"))) return -1; + if (!(npy_um_str_axes = PyUnicode_InternFromString("axes"))) return -1; + if (!(npy_um_str_axis = PyUnicode_InternFromString("axis"))) return -1; + if (!(npy_um_str_keepdims = PyUnicode_InternFromString("keepdims"))) return -1; + if (!(npy_um_str_casting = PyUnicode_InternFromString("casting"))) return -1; + if (!(npy_um_str_order = PyUnicode_InternFromString("order"))) return -1; + if (!(npy_um_str_dtype = PyUnicode_InternFromString("dtype"))) return -1; + if (!(npy_um_str_subok = PyUnicode_InternFromString("subok"))) return -1; + if (!(npy_um_str_signature = PyUnicode_InternFromString("signature"))) return -1; + if (!(npy_um_str_sig = PyUnicode_InternFromString("sig"))) return -1; + if (!(npy_um_str_extobj = PyUnicode_InternFromString("extobj"))) return -1; + if (!(npy_um_str_array_prepare = PyUnicode_InternFromString("__array_prepare__"))) return -1; + if (!(npy_um_str_array_wrap = PyUnicode_InternFromString("__array_wrap__"))) return -1; + if (!(npy_um_str_array_finalize = PyUnicode_InternFromString("__array_finalize__"))) return -1; + if (!(npy_um_str_ufunc = PyUnicode_InternFromString("__array_ufunc__"))) return -1; + if (!(npy_um_str_pyvals_name = PyUnicode_InternFromString(UFUNC_PYVALS_NAME))) return -1; return 0; } -- cgit v1.2.1 From 196d633cb19274ddffb4ba42928126fd15e3282d Mon Sep 17 00:00:00 2001 From: Raghav Khanna <37894186+raghavkhanna18@users.noreply.github.com> Date: Mon, 24 Aug 2020 08:43:05 +0800 Subject: MAINT: Chain some exceptions in arraysetops. (#17132) * MAINT: chain type error line 303 arraysetops.py * MAINT: chain axis error line 281 arraysetops.py * Revert "MAINT: chain axis error line 281 arraysetops.py" This reverts commit f6e1f544bebb7e5346cb83bdac9756be6ee9f4f6. * MAINT: chain axis error line 281 arraysetops.py to be from None --- numpy/lib/arraysetops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index df9a110c5..6a2ad004c 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -278,7 +278,7 @@ def unique(ar, return_index=False, return_inverse=False, ar = np.moveaxis(ar, axis, 0) except np.AxisError: # this removes the "axis1" or "axis2" prefix from the error message - raise np.AxisError(axis, ar.ndim) + raise np.AxisError(axis, ar.ndim) from None # Must reshape to a contiguous 2D array for this to work... orig_shape, orig_dtype = ar.shape, ar.dtype @@ -300,10 +300,10 @@ def unique(ar, return_index=False, return_inverse=False, # array with shape `(len(ar),)`. Because `dtype` in this case has # itemsize 0, the total size of the result is still 0 bytes. consolidated = np.empty(len(ar), dtype=dtype) - except TypeError: + except TypeError as e: # There's no good way to do this for object arrays, etc... msg = 'The axis argument to unique is not supported for dtype {dt}' - raise TypeError(msg.format(dt=ar.dtype)) + raise TypeError(msg.format(dt=ar.dtype)) from e def reshape_uniq(uniq): n = len(uniq) -- cgit v1.2.1 From 117b3f4a293c689777d3cd445969ca99acb01cfe Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 24 Aug 2020 09:19:08 +0100 Subject: MAINT: Replace PyInt macros with their PyLong replacement Unlike the others, PyInt_Check has a special meaning distinc from PyLong_Check, so only the places where the difference doesn't matter have been updated here. --- numpy/core/src/multiarray/buffer.c | 2 +- numpy/core/src/multiarray/calculation.c | 4 +-- numpy/core/src/multiarray/common.c | 7 ++-- numpy/core/src/multiarray/compiled_base.c | 2 +- numpy/core/src/multiarray/conversion_utils.c | 2 +- numpy/core/src/multiarray/convert.c | 2 +- numpy/core/src/multiarray/convert_datatype.c | 6 ++-- numpy/core/src/multiarray/ctors.c | 14 ++++---- numpy/core/src/multiarray/datetime.c | 31 ++++++++--------- numpy/core/src/multiarray/datetime_busdaycal.c | 2 +- numpy/core/src/multiarray/descriptor.c | 40 +++++++++++----------- numpy/core/src/multiarray/flagsobject.c | 2 +- numpy/core/src/multiarray/getset.c | 14 ++++---- numpy/core/src/multiarray/iterators.c | 8 ++--- numpy/core/src/multiarray/methods.c | 4 +-- numpy/core/src/multiarray/multiarraymodule.c | 14 ++++---- numpy/core/src/multiarray/nditer_pywrap.c | 30 ++++++++-------- numpy/core/src/multiarray/number.c | 4 +-- numpy/core/src/multiarray/refcount.c | 2 +- numpy/core/src/multiarray/scalarapi.c | 4 +-- numpy/core/src/multiarray/shape.c | 2 +- numpy/core/src/multiarray/usertypes.c | 2 +- numpy/core/src/umath/extobj.c | 6 ++-- numpy/core/src/umath/ufunc_object.c | 24 ++++++------- numpy/core/src/umath/ufunc_type_resolution.c | 8 ++--- numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c | 8 ++--- 26 files changed, 119 insertions(+), 125 deletions(-) diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 8b482dc03..25bb2d195 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -267,7 +267,7 @@ _buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str, child = (PyArray_Descr*)PyTuple_GetItem(item, 0); offset_obj = PyTuple_GetItem(item, 1); - new_offset = PyInt_AsLong(offset_obj); + new_offset = PyLong_AsLong(offset_obj); if (error_converting(new_offset)) { return -1; } diff --git a/numpy/core/src/multiarray/calculation.c b/numpy/core/src/multiarray/calculation.c index 92ab75053..43d88271b 100644 --- a/numpy/core/src/multiarray/calculation.c +++ b/numpy/core/src/multiarray/calculation.c @@ -392,7 +392,7 @@ __New_PyArray_Std(PyArrayObject *self, int axis, int rtype, PyArrayObject *out, else { val = PyArray_DIM(arrnew,i); } - PyTuple_SET_ITEM(newshape, i, PyInt_FromLong((long)val)); + PyTuple_SET_ITEM(newshape, i, PyLong_FromLong((long)val)); } arr2 = (PyArrayObject *)PyArray_Reshape(arr1, newshape); Py_DECREF(arr1); @@ -1023,7 +1023,7 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o if (min != NULL) { if (PyArray_ISUNSIGNED(self)) { int cmp; - zero = PyInt_FromLong(0); + zero = PyLong_FromLong(0); cmp = PyObject_RichCompareBool(min, zero, Py_LT); if (cmp == -1) { Py_DECREF(zero); diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 2abc79167..3d3ac7709 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -169,7 +169,7 @@ NPY_NO_EXPORT int _zerofill(PyArrayObject *ret) { if (PyDataType_REFCHK(PyArray_DESCR(ret))) { - PyObject *zero = PyInt_FromLong(0); + PyObject *zero = PyLong_FromLong(0); PyArray_FillObjectArray(ret, zero); Py_DECREF(zero); if (PyErr_Occurred()) { @@ -373,10 +373,7 @@ _unpack_field(PyObject *value, PyArray_Descr **descr, npy_intp *offset) *descr = (PyArray_Descr *)PyTuple_GET_ITEM(value, 0); off = PyTuple_GET_ITEM(value, 1); - if (PyInt_Check(off)) { - *offset = PyInt_AsSsize_t(off); - } - else if (PyLong_Check(off)) { + if (PyLong_Check(off)) { *offset = PyLong_AsSsize_t(off); } else { diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index a8e4aa789..061db2250 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -249,7 +249,7 @@ arr__monotonicity(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) NPY_END_THREADS Py_DECREF(arr_x); - return PyInt_FromLong(monotonic); + return PyLong_FromLong(monotonic); } /* diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index e41fdc8f1..6ac7a2088 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -1152,7 +1152,7 @@ PyArray_IntTupleFromIntp(int len, npy_intp const *vals) } for (i = 0; i < len; i++) { #if NPY_SIZEOF_INTP <= NPY_SIZEOF_LONG - PyObject *o = PyInt_FromLong((long) vals[i]); + PyObject *o = PyLong_FromLong((long) vals[i]); #else PyObject *o = PyLong_FromLongLong((npy_longlong) vals[i]); #endif diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index e7cbeaa77..41a10afdc 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -403,7 +403,7 @@ PyArray_FillWithScalar(PyArrayObject *arr, PyObject *obj) } } /* Python integer */ - else if (PyLong_Check(obj) || PyInt_Check(obj)) { + else if (PyLong_Check(obj)) { /* Try long long before unsigned long long */ npy_longlong ll_v = PyLong_AsLongLong(obj); if (error_converting(ll_v)) { diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 94cd1e5fa..bd038c53a 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -92,7 +92,7 @@ PyArray_GetCastFunc(PyArray_Descr *descr, int type_num) PyObject *key; PyObject *cobj; - key = PyInt_FromLong(type_num); + key = PyLong_FromLong(type_num); cobj = PyDict_GetItem(obj, key); Py_DECREF(key); if (cobj && NpyCapsule_Check(cobj)) { @@ -1989,7 +1989,7 @@ PyArray_Zero(PyArrayObject *arr) } if (zero_obj == NULL) { - zero_obj = PyInt_FromLong((long) 0); + zero_obj = PyLong_FromLong((long) 0); if (zero_obj == NULL) { return NULL; } @@ -2035,7 +2035,7 @@ PyArray_One(PyArrayObject *arr) } if (one_obj == NULL) { - one_obj = PyInt_FromLong((long) 1); + one_obj = PyLong_FromLong((long) 1); if (one_obj == NULL) { return NULL; } diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 15824e9e2..3ff397817 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -300,12 +300,12 @@ _update_descr_and_dimensions(PyArray_Descr **des, npy_intp *newdims, } if (tuple) { for (i = 0; i < numnew; i++) { - mydim[i] = (npy_intp) PyInt_AsLong( + mydim[i] = (npy_intp) PyLong_AsLong( PyTuple_GET_ITEM(old->subarray->shape, i)); } } else { - mydim[0] = (npy_intp) PyInt_AsLong(old->subarray->shape); + mydim[0] = (npy_intp) PyLong_AsLong(old->subarray->shape); } if (newstrides) { @@ -1836,8 +1836,6 @@ _is_default_descr(PyObject *descr, PyObject *typestr) { return ret; } -#define PyIntOrLong_Check(obj) (PyInt_Check(obj) || PyLong_Check(obj)) - /*NUMPY_API*/ NPY_NO_EXPORT PyObject * PyArray_FromInterface(PyObject *origin) @@ -1999,7 +1997,7 @@ PyArray_FromInterface(PyObject *origin) goto fail; } } - else if (PyIntOrLong_Check(dataptr)) { + else if (PyLong_Check(dataptr)) { data = PyLong_AsVoidPtr(dataptr); } else { @@ -2750,7 +2748,7 @@ _calc_length(PyObject *start, PyObject *stop, PyObject *step, PyObject **next, i return -1; } - zero = PyInt_FromLong(0); + zero = PyLong_FromLong(0); if (!zero) { Py_DECREF(*next); *next = NULL; @@ -2895,14 +2893,14 @@ PyArray_ArangeObj(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr Py_INCREF(dtype); } if (!step || step == Py_None) { - step = PyInt_FromLong(1); + step = PyLong_FromLong(1); } else { Py_XINCREF(step); } if (!stop || stop == Py_None) { stop = start; - start = PyInt_FromLong(0); + start = PyLong_FromLong(0); } else { Py_INCREF(start); diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 8f3948c23..3649bbe4c 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1749,7 +1749,7 @@ convert_datetime_metadata_to_tuple(PyArray_DatetimeMetaData *meta) PyTuple_SET_ITEM(dt_tuple, 0, PyUString_FromString(_datetime_strings[meta->base])); PyTuple_SET_ITEM(dt_tuple, 1, - PyInt_FromLong(meta->num)); + PyLong_FromLong(meta->num)); return dt_tuple; } @@ -1812,7 +1812,7 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, Py_DECREF(unit_str); /* Convert the values to longs */ - out_meta->num = PyInt_AsLong(PyTuple_GET_ITEM(tuple, 1)); + out_meta->num = PyLong_AsLong(PyTuple_GET_ITEM(tuple, 1)); if (error_converting(out_meta->num)) { return -1; } @@ -1868,7 +1868,7 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, return -1; } } - den = PyInt_AsLong(PyTuple_GET_ITEM(tuple, 2)); + den = PyLong_AsLong(PyTuple_GET_ITEM(tuple, 2)); if (error_converting(den)) { return -1; } @@ -2108,7 +2108,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->year = PyInt_AsLong(tmp); + out->year = PyLong_AsLong(tmp); if (error_converting(out->year)) { Py_DECREF(tmp); return -1; @@ -2120,7 +2120,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->month = PyInt_AsLong(tmp); + out->month = PyLong_AsLong(tmp); if (error_converting(out->month)) { Py_DECREF(tmp); return -1; @@ -2132,7 +2132,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->day = PyInt_AsLong(tmp); + out->day = PyLong_AsLong(tmp); if (error_converting(out->day)) { Py_DECREF(tmp); return -1; @@ -2166,7 +2166,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->hour = PyInt_AsLong(tmp); + out->hour = PyLong_AsLong(tmp); if (error_converting(out->hour)) { Py_DECREF(tmp); return -1; @@ -2178,7 +2178,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->min = PyInt_AsLong(tmp); + out->min = PyLong_AsLong(tmp); if (error_converting(out->min)) { Py_DECREF(tmp); return -1; @@ -2190,7 +2190,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->sec = PyInt_AsLong(tmp); + out->sec = PyLong_AsLong(tmp); if (error_converting(out->sec)) { Py_DECREF(tmp); return -1; @@ -2202,7 +2202,7 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, if (tmp == NULL) { return -1; } - out->us = PyInt_AsLong(tmp); + out->us = PyLong_AsLong(tmp); if (error_converting(out->us)) { Py_DECREF(tmp); return -1; @@ -2394,7 +2394,7 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, return 0; } /* Do no conversion on raw integers */ - else if (PyInt_Check(obj) || PyLong_Check(obj)) { + else if (PyLong_Check(obj)) { /* Don't allow conversion from an integer without specifying a unit */ if (meta->base == NPY_FR_ERROR || meta->base == NPY_FR_GENERIC) { PyErr_SetString(PyExc_ValueError, "Converting an integer to a " @@ -2595,7 +2595,7 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, } } /* Do no conversion on raw integers */ - else if (PyInt_Check(obj) || PyLong_Check(obj)) { + else if (PyLong_Check(obj)) { /* Use the default unit if none was specified */ if (meta->base == NPY_FR_ERROR) { meta->base = NPY_DATETIME_DEFAULTUNIT; @@ -2699,7 +2699,7 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, if (tmp == NULL) { return -1; } - seconds = PyInt_AsLong(tmp); + seconds = PyLong_AsLong(tmp); if (error_converting(seconds)) { Py_DECREF(tmp); return -1; @@ -2711,7 +2711,7 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, if (tmp == NULL) { return -1; } - useconds = PyInt_AsLong(tmp); + useconds = PyLong_AsLong(tmp); if (error_converting(useconds)) { Py_DECREF(tmp); return -1; @@ -3320,8 +3320,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, type_nums[2] = NPY_TIMEDELTA; } else { - if (PyInt_Check(objs[1]) || - PyLong_Check(objs[1]) || + if (PyLong_Check(objs[1]) || PyArray_IsScalar(objs[1], Integer) || is_any_numpy_timedelta(objs[1])) { type_nums[1] = NPY_TIMEDELTA; diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 6936a803f..2374eaa63 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -168,7 +168,7 @@ invalid_weekmask_string: return 0; } - val = PyInt_AsLong(f); + val = PyLong_AsLong(f); if (error_converting(val)) { Py_DECREF(f); Py_DECREF(obj); diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 67d57975b..f47f0ce06 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -386,7 +386,7 @@ _convert_from_tuple(PyObject *obj, int align) } for (int i=0; i < shape.len; i++) { PyTuple_SET_ITEM(newdescr->subarray->shape, i, - PyInt_FromLong((long)shape.ptr[i])); + PyLong_FromLong((long)shape.ptr[i])); if (PyTuple_GET_ITEM(newdescr->subarray->shape, i) == NULL) { Py_DECREF(newdescr); @@ -537,7 +537,7 @@ _convert_from_array_descr(PyObject *obj, int align) goto fail; } PyTuple_SET_ITEM(tup, 0, (PyObject *)conv); - PyTuple_SET_ITEM(tup, 1, PyInt_FromLong((long) totalsize)); + PyTuple_SET_ITEM(tup, 1, PyLong_FromLong((long) totalsize)); /* * Title can be "meta-data". Only insert it @@ -660,7 +660,7 @@ _convert_from_list(PyObject *obj, int align) } maxalign = PyArray_MAX(maxalign, _align); } - PyObject *size_obj = PyInt_FromLong((long) totalsize); + PyObject *size_obj = PyLong_FromLong((long) totalsize); if (!size_obj) { Py_DECREF(conv); goto fail; @@ -1112,7 +1112,7 @@ _convert_from_dict(PyObject *obj, int align) /* Build item to insert (descr, offset, [title])*/ int len = 2; PyObject *title = NULL; - PyObject *ind = PyInt_FromLong(i); + PyObject *ind = PyLong_FromLong(i); if (titles) { title=PyObject_GetItem(titles, ind); if (title && title != Py_None) { @@ -1166,7 +1166,7 @@ _convert_from_dict(PyObject *obj, int align) goto fail; } - PyTuple_SET_ITEM(tup, 1, PyInt_FromLong(offset)); + PyTuple_SET_ITEM(tup, 1, PyLong_FromLong(offset)); /* Flag whether the fields are specified out of order */ if (offset < totalsize) { has_out_of_order_fields = 1; @@ -1190,7 +1190,7 @@ _convert_from_dict(PyObject *obj, int align) if (align && _align > 1) { totalsize = NPY_NEXT_ALIGNED_OFFSET(totalsize, _align); } - PyTuple_SET_ITEM(tup, 1, PyInt_FromLong(totalsize)); + PyTuple_SET_ITEM(tup, 1, PyLong_FromLong(totalsize)); totalsize += newdescr->elsize; } if (len == 3) { @@ -1950,7 +1950,7 @@ arraydescr_ndim_get(PyArray_Descr *self) Py_ssize_t ndim; if (!PyDataType_HASSUBARRAY(self)) { - return PyInt_FromLong(0); + return PyLong_FromLong(0); } /* @@ -1958,7 +1958,7 @@ arraydescr_ndim_get(PyArray_Descr *self) * for tuple argument */ ndim = PyTuple_Size(self->subarray->shape); - return PyInt_FromLong(ndim); + return PyLong_FromLong(ndim); } @@ -2010,7 +2010,7 @@ arraydescr_isbuiltin_get(PyArray_Descr *self) if (PyTypeNum_ISUSERDEF(self->type_num)) { val = 2; } - return PyInt_FromLong(val); + return PyLong_FromLong(val); } static int @@ -2391,11 +2391,11 @@ _get_pickleabletype_from_datetime_metadata(PyArray_Descr *dtype) PyTuple_SET_ITEM(dt_tuple, 0, PyBytes_FromString(_datetime_strings[meta->base])); PyTuple_SET_ITEM(dt_tuple, 1, - PyInt_FromLong(meta->num)); + PyLong_FromLong(meta->num)); PyTuple_SET_ITEM(dt_tuple, 2, - PyInt_FromLong(1)); + PyLong_FromLong(1)); PyTuple_SET_ITEM(dt_tuple, 3, - PyInt_FromLong(1)); + PyLong_FromLong(1)); PyTuple_SET_ITEM(ret, 1, dt_tuple); @@ -2468,7 +2468,7 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) if (PyDataType_ISDATETIME(self)) { PyObject *newobj; state = PyTuple_New(9); - PyTuple_SET_ITEM(state, 0, PyInt_FromLong(version)); + PyTuple_SET_ITEM(state, 0, PyLong_FromLong(version)); /* * newobj is a tuple of the Python metadata dictionary * and tuple of date_time info (str, num) @@ -2483,13 +2483,13 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) } else if (self->metadata) { state = PyTuple_New(9); - PyTuple_SET_ITEM(state, 0, PyInt_FromLong(version)); + PyTuple_SET_ITEM(state, 0, PyLong_FromLong(version)); Py_INCREF(self->metadata); PyTuple_SET_ITEM(state, 8, self->metadata); } else { /* Use version 3 pickle format */ state = PyTuple_New(8); - PyTuple_SET_ITEM(state, 0, PyInt_FromLong(3)); + PyTuple_SET_ITEM(state, 0, PyLong_FromLong(3)); } PyTuple_SET_ITEM(state, 1, PyUString_FromFormat("%c", endian)); @@ -2516,9 +2516,9 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) elsize = -1; alignment = -1; } - PyTuple_SET_ITEM(state, 5, PyInt_FromLong(elsize)); - PyTuple_SET_ITEM(state, 6, PyInt_FromLong(alignment)); - PyTuple_SET_ITEM(state, 7, PyInt_FromLong(self->flags)); + PyTuple_SET_ITEM(state, 5, PyLong_FromLong(elsize)); + PyTuple_SET_ITEM(state, 6, PyLong_FromLong(alignment)); + PyTuple_SET_ITEM(state, 7, PyLong_FromLong(self->flags)); PyTuple_SET_ITEM(ret, 2, state); return ret; @@ -2628,7 +2628,7 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) default: /* raise an error */ if (PyTuple_GET_SIZE(PyTuple_GET_ITEM(args,0)) > 5) { - version = PyInt_AsLong(PyTuple_GET_ITEM(args, 0)); + version = PyLong_AsLong(PyTuple_GET_ITEM(args, 0)); } else { version = -1; @@ -2651,7 +2651,7 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) if (version == 1 || version == 0) { if (fields != Py_None) { PyObject *key, *list; - key = PyInt_FromLong(-1); + key = PyLong_FromLong(-1); list = PyDict_GetItemWithError(fields, key); if (!list) { if (!PyErr_Occurred()) { diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index d5f24e75a..bec0523d5 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -307,7 +307,7 @@ arrayflags_farray_get(PyArrayFlagsObject *self) static PyObject * arrayflags_num_get(PyArrayFlagsObject *self) { - return PyInt_FromLong(self->flags); + return PyLong_FromLong(self->flags); } /* relies on setflags order being write, align, uic */ diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 9066f52a8..c8533539b 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -28,7 +28,7 @@ static PyObject * array_ndim_get(PyArrayObject *self) { - return PyInt_FromLong(PyArray_NDIM(self)); + return PyLong_FromLong(PyArray_NDIM(self)); } static PyObject * @@ -318,7 +318,7 @@ array_interface_get(PyArrayObject *self) return NULL; } - obj = PyInt_FromLong(3); + obj = PyLong_FromLong(3); ret = PyDict_SetItemString(dict, "version", obj); Py_DECREF(obj); if (ret < 0) { @@ -413,7 +413,7 @@ array_data_set(PyArrayObject *self, PyObject *op) static PyObject * array_itemsize_get(PyArrayObject *self) { - return PyInt_FromLong((long) PyArray_DESCR(self)->elsize); + return PyLong_FromLong((long) PyArray_DESCR(self)->elsize); } static PyObject * @@ -421,13 +421,13 @@ array_size_get(PyArrayObject *self) { npy_intp size=PyArray_SIZE(self); #if NPY_SIZEOF_INTP <= NPY_SIZEOF_LONG - return PyInt_FromLong((long) size); + return PyLong_FromLong((long) size); #else if (size > NPY_MAX_LONG || size < NPY_MIN_LONG) { return PyLong_FromLongLong(size); } else { - return PyInt_FromLong((long) size); + return PyLong_FromLong((long) size); } #endif } @@ -437,13 +437,13 @@ array_nbytes_get(PyArrayObject *self) { npy_intp nbytes = PyArray_NBYTES(self); #if NPY_SIZEOF_INTP <= NPY_SIZEOF_LONG - return PyInt_FromLong((long) nbytes); + return PyLong_FromLong((long) nbytes); #else if (nbytes > NPY_MAX_LONG || nbytes < NPY_MIN_LONG) { return PyLong_FromLongLong(nbytes); } else { - return PyInt_FromLong((long) nbytes); + return PyLong_FromLong((long) nbytes); } #endif } diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index ac5b90400..96f501c55 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -1411,10 +1411,10 @@ static PyObject * arraymultiter_size_get(PyArrayMultiIterObject *self) { #if NPY_SIZEOF_INTP <= NPY_SIZEOF_LONG - return PyInt_FromLong((long) self->size); + return PyLong_FromLong((long) self->size); #else if (self->size < NPY_MAX_LONG) { - return PyInt_FromLong((long) self->size); + return PyLong_FromLong((long) self->size); } else { return PyLong_FromLongLong((npy_longlong) self->size); @@ -1426,10 +1426,10 @@ static PyObject * arraymultiter_index_get(PyArrayMultiIterObject *self) { #if NPY_SIZEOF_INTP <= NPY_SIZEOF_LONG - return PyInt_FromLong((long) self->index); + return PyLong_FromLong((long) self->index); #else if (self->size < NPY_MAX_LONG) { - return PyInt_FromLong((long) self->index); + return PyLong_FromLong((long) self->index); } else { return PyLong_FromLongLong((npy_longlong) self->index); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index e0b36e80f..0519434e8 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1676,7 +1676,7 @@ array_reduce(PyArrayObject *self, PyObject *NPY_UNUSED(args)) Py_BuildValue("ONc", (PyObject *)Py_TYPE(self), Py_BuildValue("(N)", - PyInt_FromLong(0)), + PyLong_FromLong(0)), /* dummy data-type */ 'b')); @@ -1701,7 +1701,7 @@ array_reduce(PyArrayObject *self, PyObject *NPY_UNUSED(args)) Py_DECREF(ret); return NULL; } - PyTuple_SET_ITEM(state, 0, PyInt_FromLong(version)); + PyTuple_SET_ITEM(state, 0, PyLong_FromLong(version)); PyTuple_SET_ITEM(state, 1, PyObject_GetAttrString((PyObject *)self, "shape")); descr = PyArray_DESCR(self); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 7c5ceb962..923469edf 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -2810,7 +2810,7 @@ array__get_ndarray_c_version(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObje if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist )) { return NULL; } - return PyInt_FromLong( (long) PyArray_GetNDArrayCVersion() ); + return PyLong_FromLong( (long) PyArray_GetNDArrayCVersion() ); } /*NUMPY_API @@ -3950,7 +3950,7 @@ normalize_axis_index(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) return NULL; } - return PyInt_FromLong(axis); + return PyLong_FromLong(axis); } @@ -4292,13 +4292,13 @@ set_flaginfo(PyObject *d) newd = PyDict_New(); #define _addnew(key, val, one) \ - PyDict_SetItemString(newd, #key, s=PyInt_FromLong(val)); \ + PyDict_SetItemString(newd, #key, s=PyLong_FromLong(val)); \ Py_DECREF(s); \ - PyDict_SetItemString(newd, #one, s=PyInt_FromLong(val)); \ + PyDict_SetItemString(newd, #one, s=PyLong_FromLong(val)); \ Py_DECREF(s) #define _addone(key, val) \ - PyDict_SetItemString(newd, #key, s=PyInt_FromLong(val)); \ + PyDict_SetItemString(newd, #key, s=PyLong_FromLong(val)); \ Py_DECREF(s) _addnew(OWNDATA, NPY_ARRAY_OWNDATA, O); @@ -4502,7 +4502,7 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { */ PyDict_SetItemString (d, "error", PyExc_Exception); - s = PyInt_FromLong(NPY_TRACE_DOMAIN); + s = PyLong_FromLong(NPY_TRACE_DOMAIN); PyDict_SetItemString(d, "tracemalloc_domain", s); Py_DECREF(s); @@ -4548,7 +4548,7 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { Py_DECREF(s); #define ADDCONST(NAME) \ - s = PyInt_FromLong(NPY_##NAME); \ + s = PyLong_FromLong(NPY_##NAME); \ PyDict_SetItemString(d, #NAME, s); \ Py_DECREF(s) diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 1c68a4803..e271906c1 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -894,7 +894,7 @@ NpyIter_NestedIters(PyObject *NPY_UNUSED(self), Py_DECREF(item); return NULL; } - axis = PyInt_AsLong(v); + axis = PyLong_AsLong(v); Py_DECREF(v); if (axis < 0 || axis >= NPY_MAXDIMS) { PyErr_SetString(PyExc_ValueError, @@ -1522,7 +1522,7 @@ static PyObject *npyiter_shape_get(NewNpyArrayIterObject *self) if (ret != NULL) { for (idim = 0; idim < ndim; ++idim) { PyTuple_SET_ITEM(ret, idim, - PyInt_FromLong(shape[idim])); + PyLong_FromLong(shape[idim])); } return ret; } @@ -1551,7 +1551,7 @@ static PyObject *npyiter_multi_index_get(NewNpyArrayIterObject *self) } for (idim = 0; idim < ndim; ++idim) { PyTuple_SET_ITEM(ret, idim, - PyInt_FromLong(multi_index[idim])); + PyLong_FromLong(multi_index[idim])); } return ret; } @@ -1605,7 +1605,7 @@ npyiter_multi_index_set(NewNpyArrayIterObject *self, PyObject *value) } for (idim = 0; idim < ndim; ++idim) { PyObject *v = PySequence_GetItem(value, idim); - multi_index[idim] = PyInt_AsLong(v); + multi_index[idim] = PyLong_AsLong(v); if (error_converting(multi_index[idim])) { Py_XDECREF(v); return -1; @@ -1641,7 +1641,7 @@ static PyObject *npyiter_index_get(NewNpyArrayIterObject *self) if (NpyIter_HasIndex(self->iter)) { npy_intp ind = *NpyIter_GetIndexPtr(self->iter); - return PyInt_FromLong(ind); + return PyLong_FromLong(ind); } else { PyErr_SetString(PyExc_ValueError, @@ -1665,7 +1665,7 @@ static int npyiter_index_set(NewNpyArrayIterObject *self, PyObject *value) if (NpyIter_HasIndex(self->iter)) { npy_intp ind; - ind = PyInt_AsLong(value); + ind = PyLong_AsLong(value); if (error_converting(ind)) { return -1; } @@ -1697,7 +1697,7 @@ static PyObject *npyiter_iterindex_get(NewNpyArrayIterObject *self) return NULL; } - return PyInt_FromLong(NpyIter_GetIterIndex(self->iter)); + return PyLong_FromLong(NpyIter_GetIterIndex(self->iter)); } static int npyiter_iterindex_set(NewNpyArrayIterObject *self, PyObject *value) @@ -1715,7 +1715,7 @@ static int npyiter_iterindex_set(NewNpyArrayIterObject *self, PyObject *value) return -1; } - iterindex = PyInt_AsLong(value); + iterindex = PyLong_AsLong(value); if (error_converting(iterindex)) { return -1; } @@ -1751,8 +1751,8 @@ static PyObject *npyiter_iterrange_get(NewNpyArrayIterObject *self) return NULL; } - PyTuple_SET_ITEM(ret, 0, PyInt_FromLong(istart)); - PyTuple_SET_ITEM(ret, 1, PyInt_FromLong(iend)); + PyTuple_SET_ITEM(ret, 0, PyLong_FromLong(istart)); + PyTuple_SET_ITEM(ret, 1, PyLong_FromLong(iend)); return ret; } @@ -1900,7 +1900,7 @@ static PyObject *npyiter_ndim_get(NewNpyArrayIterObject *self) return NULL; } - return PyInt_FromLong(NpyIter_GetNDim(self->iter)); + return PyLong_FromLong(NpyIter_GetNDim(self->iter)); } static PyObject *npyiter_nop_get(NewNpyArrayIterObject *self) @@ -1911,7 +1911,7 @@ static PyObject *npyiter_nop_get(NewNpyArrayIterObject *self) return NULL; } - return PyInt_FromLong(NpyIter_GetNOp(self->iter)); + return PyLong_FromLong(NpyIter_GetNOp(self->iter)); } static PyObject *npyiter_itersize_get(NewNpyArrayIterObject *self) @@ -1922,7 +1922,7 @@ static PyObject *npyiter_itersize_get(NewNpyArrayIterObject *self) return NULL; } - return PyInt_FromLong(NpyIter_GetIterSize(self->iter)); + return PyLong_FromLong(NpyIter_GetIterSize(self->iter)); } static PyObject *npyiter_finished_get(NewNpyArrayIterObject *self) @@ -2221,7 +2221,7 @@ npyiter_subscript(NewNpyArrayIterObject *self, PyObject *op) return NULL; } - if (PyInt_Check(op) || PyLong_Check(op) || + if (PyLong_Check(op) || (PyIndex_Check(op) && !PySequence_Check(op))) { npy_intp i = PyArray_PyIntAsIntp(op); if (error_converting(i)) { @@ -2270,7 +2270,7 @@ npyiter_ass_subscript(NewNpyArrayIterObject *self, PyObject *op, return -1; } - if (PyInt_Check(op) || PyLong_Check(op) || + if (PyLong_Check(op) || (PyIndex_Check(op) && !PySequence_Check(op))) { npy_intp i = PyArray_PyIntAsIntp(op); if (error_converting(i)) { diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 19ac7d7f9..b4f690920 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -398,7 +398,7 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) const int optimize_fpexps = 1; if (PyInt_Check(o2)) { - *out_exponent = (double)PyInt_AsLong(o2); + *out_exponent = (double)PyLong_AsLong(o2); return NPY_INTPOS_SCALAR; } if (optimize_fpexps && PyFloat_Check(o2)) { @@ -448,7 +448,7 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) } return NPY_NOSCALAR; } - val = PyInt_AsSsize_t(value); + val = PyLong_AsSsize_t(value); if (error_converting(val)) { PyErr_Clear(); return NPY_NOSCALAR; diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index c869b5eea..985d187fa 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -292,7 +292,7 @@ static void _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) { if (!PyDataType_FLAGCHK(dtype, NPY_ITEM_REFCOUNT)) { - if ((obj == Py_None) || (PyInt_Check(obj) && PyInt_AsLong(obj)==0)) { + if ((obj == Py_None) || (PyInt_Check(obj) && PyLong_AsLong(obj)==0)) { return; } else { diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index 6f3d102a4..4dee259f8 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -380,7 +380,7 @@ PyArray_ScalarFromObject(PyObject *object) } /* * Booleans in Python are implemented as a subclass of integers, - * so PyBool_Check must be called before PyInt_Check. + * so PyBool_Check must be called before PyLong_Check. */ if (PyBool_Check(object)) { if (object == Py_True) { @@ -395,7 +395,7 @@ PyArray_ScalarFromObject(PyObject *object) if (ret == NULL) { return NULL; } - PyArrayScalar_VAL(ret, Long) = PyInt_AS_LONG(object); + PyArrayScalar_VAL(ret, Long) = PyLong_AsLong(object); } else if (PyFloat_Check(object)) { ret = PyArrayScalar_New(Double); diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 30507112d..73bb7933f 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -133,7 +133,7 @@ PyArray_Resize(PyArrayObject *self, PyArray_Dims *newshape, int refcheck, if (newnbytes > oldnbytes && PyArray_ISWRITEABLE(self)) { /* Fill new memory with zeros */ if (PyDataType_FLAGCHK(PyArray_DESCR(self), NPY_ITEM_REFCOUNT)) { - PyObject *zero = PyInt_FromLong(0); + PyObject *zero = PyLong_FromLong(0); char *optr; optr = PyArray_BYTES(self) + oldnbytes; npy_intp n_new = newsize - oldsize; diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 0c8d49970..b97f0f8b8 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -268,7 +268,7 @@ PyArray_RegisterCastFunc(PyArray_Descr *descr, int totype, return -1; } } - key = PyInt_FromLong(totype); + key = PyLong_FromLong(totype); if (PyErr_Occurred()) { return -1; } diff --git a/numpy/core/src/umath/extobj.c b/numpy/core/src/umath/extobj.c index 3404a0c6a..4a953410a 100644 --- a/numpy/core/src/umath/extobj.c +++ b/numpy/core/src/umath/extobj.c @@ -110,7 +110,7 @@ _error_handler(int method, PyObject *errobj, char *errtype, int retstatus, int * goto fail; } args = Py_BuildValue("NN", PyUString_FromString(errtype), - PyInt_FromLong((long) retstatus)); + PyLong_FromLong((long) retstatus)); if (args == NULL) { goto fail; } @@ -212,7 +212,7 @@ _extract_pyvals(PyObject *ref, const char *name, int *bufsize, } if (bufsize != NULL) { - *bufsize = PyInt_AsLong(PyList_GET_ITEM(ref, 0)); + *bufsize = PyLong_AsLong(PyList_GET_ITEM(ref, 0)); if (error_converting(*bufsize)) { return -1; } @@ -229,7 +229,7 @@ _extract_pyvals(PyObject *ref, const char *name, int *bufsize, } if (errmask != NULL) { - *errmask = PyInt_AsLong(PyList_GET_ITEM(ref, 1)); + *errmask = PyLong_AsLong(PyList_GET_ITEM(ref, 1)); if (*errmask < 0) { if (PyErr_Occurred()) { return -1; diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index b47ccd291..005556fb6 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -2432,15 +2432,15 @@ _get_identity(PyUFuncObject *ufunc, npy_bool *reorderable) { switch(ufunc->identity) { case PyUFunc_One: *reorderable = 1; - return PyInt_FromLong(1); + return PyLong_FromLong(1); case PyUFunc_Zero: *reorderable = 1; - return PyInt_FromLong(0); + return PyLong_FromLong(0); case PyUFunc_MinusOne: *reorderable = 1; - return PyInt_FromLong(-1); + return PyLong_FromLong(-1); case PyUFunc_ReorderableNone: *reorderable = 1; @@ -3326,7 +3326,7 @@ get_binary_op_function(PyUFuncObject *ufunc, int *otype, /* If the type is custom and there are userloops, search for it here */ if (ufunc->userloops != NULL && PyTypeNum_ISUSERDEF(*otype)) { PyObject *key, *obj; - key = PyInt_FromLong(*otype); + key = PyLong_FromLong(*otype); if (key == NULL) { return -1; } @@ -4834,8 +4834,8 @@ ufunc_geterr(PyObject *NPY_UNUSED(dummy), PyObject *args) if (res == NULL) { return NULL; } - PyList_SET_ITEM(res, 0, PyInt_FromLong(NPY_BUFSIZE)); - PyList_SET_ITEM(res, 1, PyInt_FromLong(UFUNC_ERR_DEFAULT)); + PyList_SET_ITEM(res, 0, PyLong_FromLong(NPY_BUFSIZE)); + PyList_SET_ITEM(res, 1, PyLong_FromLong(UFUNC_ERR_DEFAULT)); PyList_SET_ITEM(res, 2, Py_None); Py_INCREF(Py_None); return res; } @@ -5155,7 +5155,7 @@ PyUFunc_RegisterLoopForDescr(PyUFuncObject *ufunc, return -1; } - key = PyInt_FromLong((long) user_dtype->type_num); + key = PyLong_FromLong((long) user_dtype->type_num); if (key == NULL) { return -1; } @@ -5257,7 +5257,7 @@ PyUFunc_RegisterLoopForType(PyUFuncObject *ufunc, if (ufunc->userloops == NULL) { ufunc->userloops = PyDict_New(); } - key = PyInt_FromLong((long) usertype); + key = PyLong_FromLong((long) usertype); if (key == NULL) { return -1; } @@ -6003,25 +6003,25 @@ ufunc_get_doc(PyUFuncObject *ufunc) static PyObject * ufunc_get_nin(PyUFuncObject *ufunc) { - return PyInt_FromLong(ufunc->nin); + return PyLong_FromLong(ufunc->nin); } static PyObject * ufunc_get_nout(PyUFuncObject *ufunc) { - return PyInt_FromLong(ufunc->nout); + return PyLong_FromLong(ufunc->nout); } static PyObject * ufunc_get_nargs(PyUFuncObject *ufunc) { - return PyInt_FromLong(ufunc->nargs); + return PyLong_FromLong(ufunc->nargs); } static PyObject * ufunc_get_ntypes(PyUFuncObject *ufunc) { - return PyInt_FromLong(ufunc->ntypes); + return PyLong_FromLong(ufunc->ntypes); } static PyObject * diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index ea20bb24f..fec3caef2 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -46,7 +46,7 @@ npy_casting_to_py_object(NPY_CASTING casting) case NPY_UNSAFE_CASTING: return PyUString_FromString("unsafe"); default: - return PyInt_FromLong(casting); + return PyLong_FromLong(casting); } } @@ -1356,7 +1356,7 @@ find_userloop(PyUFuncObject *ufunc, last_userdef = type_num; - key = PyInt_FromLong(type_num); + key = PyLong_FromLong(type_num); if (key == NULL) { return -1; } @@ -1764,7 +1764,7 @@ linear_search_userloop_type_resolver(PyUFuncObject *self, last_userdef = type_num; - key = PyInt_FromLong(type_num); + key = PyLong_FromLong(type_num); if (key == NULL) { return -1; } @@ -1831,7 +1831,7 @@ type_tuple_userloop_type_resolver(PyUFuncObject *self, last_userdef = type_num; - key = PyInt_FromLong(type_num); + key = PyLong_FromLong(type_num); if (key == NULL) { return -1; } diff --git a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c index 0db33e714..8b089d334 100644 --- a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c +++ b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c @@ -55,7 +55,7 @@ static PyObject *f2py_rout_wrap_call(PyObject *capi_self, if (tmp == NULL) { goto fail; } - dims[i] = (npy_intp)PyInt_AsLong(tmp); + dims[i] = (npy_intp)PyLong_AsLong(tmp); Py_DECREF(tmp); if (dims[i] == -1 && PyErr_Occurred()) { goto fail; @@ -107,8 +107,8 @@ static PyObject *f2py_rout_wrap_attrs(PyObject *capi_self, dimensions = PyTuple_New(PyArray_NDIM(arr)); strides = PyTuple_New(PyArray_NDIM(arr)); for (i=0;i Date: Mon, 24 Aug 2020 11:17:23 +0100 Subject: BUG: Fix incorrect cython definition of npy_cfloat The definition in the numpy headers is a pair of `float`s not `double`s. --- numpy/__init__.cython-30.pxd | 4 ++-- numpy/__init__.pxd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/numpy/__init__.cython-30.pxd b/numpy/__init__.cython-30.pxd index 55097888c..4d9ec1fed 100644 --- a/numpy/__init__.cython-30.pxd +++ b/numpy/__init__.cython-30.pxd @@ -329,8 +329,8 @@ cdef extern from "numpy/arrayobject.h": ctypedef long double npy_float128 ctypedef struct npy_cfloat: - double real - double imag + float real + float imag ctypedef struct npy_cdouble: double real diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index 206dbe8b9..bf4298e59 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -290,8 +290,8 @@ cdef extern from "numpy/arrayobject.h": ctypedef long double npy_float128 ctypedef struct npy_cfloat: - double real - double imag + float real + float imag ctypedef struct npy_cdouble: double real -- cgit v1.2.1 From 69898992b339e4614780f626ed07c4244a08648b Mon Sep 17 00:00:00 2001 From: Bas van Beek <43369155+BvB93@users.noreply.github.com> Date: Mon, 24 Aug 2020 13:27:30 +0200 Subject: ENH: Add annotations to the last 8 functions in numpy.core.fromnumeric (#16729) * Added annotations for 8 new functions The 8 respective functions, all located in `np.core.fromnumeric`, consist of: * `prod` * `cumprod` * `ndim` * `size` * `around` * `mean` * `std` * `var` Co-authored-by: Eric Wieser --- numpy/__init__.pyi | 111 +++++++++++++++++++++++++++++++ numpy/tests/typing/fail/fromnumeric.py | 28 ++++++++ numpy/tests/typing/pass/fromnumeric.py | 75 +++++++++++++++++++++ numpy/tests/typing/reveal/fromnumeric.py | 73 ++++++++++++++++++++ 4 files changed, 287 insertions(+) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index fad5e1774..c6cc94440 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1237,3 +1237,114 @@ def amin( initial: _NumberLike = ..., where: _ArrayLikeBool = ..., ) -> Union[number, ndarray]: ... + +# TODO: `np.prod()``: For object arrays `initial` does not necessarily +# have to be a numerical scalar. +# The only requirement is that it is compatible +# with the `.__mul__()` method(s) of the passed array's elements. + +# Note that the same situation holds for all wrappers around +# `np.ufunc.reduce`, e.g. `np.sum()` (`.__add__()`). + +@overload +def prod( + a: _Number, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> _Number: ... +@overload +def prod( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> number: ... +@overload +def prod( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., + initial: _NumberLike = ..., + where: _ArrayLikeBool = ..., +) -> Union[number, ndarray]: ... +def cumprod( + a: ArrayLike, + axis: Optional[int] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., +) -> ndarray: ... +def ndim(a: ArrayLike) -> int: ... +def size(a: ArrayLike, axis: Optional[int] = ...) -> int: ... +@overload +def around( + a: _Number, decimals: int = ..., out: Optional[ndarray] = ... +) -> _Number: ... +@overload +def around( + a: _NumberLike, decimals: int = ..., out: Optional[ndarray] = ... +) -> number: ... +@overload +def around( + a: ArrayLike, decimals: int = ..., out: Optional[ndarray] = ... +) -> ndarray: ... +@overload +def mean( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def mean( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def std( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def std( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + ddof: int = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... +@overload +def var( + a: ArrayLike, + axis: None = ..., + dtype: DtypeLike = ..., + out: None = ..., + ddof: int = ..., + keepdims: Literal[False] = ..., +) -> number: ... +@overload +def var( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DtypeLike = ..., + out: Optional[ndarray] = ..., + ddof: int = ..., + keepdims: bool = ..., +) -> Union[number, ndarray]: ... diff --git a/numpy/tests/typing/fail/fromnumeric.py b/numpy/tests/typing/fail/fromnumeric.py index 66f8a89d0..c9156895d 100644 --- a/numpy/tests/typing/fail/fromnumeric.py +++ b/numpy/tests/typing/fail/fromnumeric.py @@ -124,3 +124,31 @@ np.amin(a, keepdims=1.0) # E: No overload variant of "amin" matches argument ty np.amin(a, out=1.0) # E: No overload variant of "amin" matches argument type np.amin(a, initial=[1.0]) # E: No overload variant of "amin" matches argument type np.amin(a, where=[1.0]) # E: List item 0 has incompatible type + +np.prod(a, axis=1.0) # E: No overload variant of "prod" matches argument type +np.prod(a, out=False) # E: No overload variant of "prod" matches argument type +np.prod(a, keepdims=1.0) # E: No overload variant of "prod" matches argument type +np.prod(a, initial=int) # E: No overload variant of "prod" matches argument type +np.prod(a, where=1.0) # E: No overload variant of "prod" matches argument type + +np.cumprod(a, axis=1.0) # E: Argument "axis" to "cumprod" has incompatible type +np.cumprod(a, out=False) # E: Argument "out" to "cumprod" has incompatible type + +np.size(a, axis=1.0) # E: Argument "axis" to "size" has incompatible type + +np.around(a, decimals=1.0) # E: No overload variant of "around" matches argument type +np.around(a, out=type) # E: No overload variant of "around" matches argument type + +np.mean(a, axis=1.0) # E: No overload variant of "mean" matches argument type +np.mean(a, out=False) # E: No overload variant of "mean" matches argument type +np.mean(a, keepdims=1.0) # E: No overload variant of "mean" matches argument type + +np.std(a, axis=1.0) # E: No overload variant of "std" matches argument type +np.std(a, out=False) # E: No overload variant of "std" matches argument type +np.std(a, ddof='test') # E: No overload variant of "std" matches argument type +np.std(a, keepdims=1.0) # E: No overload variant of "std" matches argument type + +np.var(a, axis=1.0) # E: No overload variant of "var" matches argument type +np.var(a, out=False) # E: No overload variant of "var" matches argument type +np.var(a, ddof='test') # E: No overload variant of "var" matches argument type +np.var(a, keepdims=1.0) # E: No overload variant of "var" matches argument type diff --git a/numpy/tests/typing/pass/fromnumeric.py b/numpy/tests/typing/pass/fromnumeric.py index d9dd45c54..9e936e684 100644 --- a/numpy/tests/typing/pass/fromnumeric.py +++ b/numpy/tests/typing/pass/fromnumeric.py @@ -10,6 +10,7 @@ B.setflags(write=False) a = np.bool_(True) b = np.float32(1.0) c = 1.0 +d = np.array(1.0, dtype=np.float32) # writeable np.take(a, 0) np.take(b, 0) @@ -183,3 +184,77 @@ np.amin(A, axis=0) np.amin(B, axis=0) np.amin(A, keepdims=True) np.amin(B, keepdims=True) + +np.prod(a) +np.prod(b) +np.prod(c) +np.prod(A) +np.prod(B) +np.prod(a, dtype=None) +np.prod(A, dtype=None) +np.prod(A, axis=0) +np.prod(B, axis=0) +np.prod(A, keepdims=True) +np.prod(B, keepdims=True) +np.prod(b, out=d) +np.prod(B, out=d) + +np.cumprod(a) +np.cumprod(b) +np.cumprod(c) +np.cumprod(A) +np.cumprod(B) + +np.ndim(a) +np.ndim(b) +np.ndim(c) +np.ndim(A) +np.ndim(B) + +np.size(a) +np.size(b) +np.size(c) +np.size(A) +np.size(B) + +np.around(a) +np.around(b) +np.around(c) +np.around(A) +np.around(B) + +np.mean(a) +np.mean(b) +np.mean(c) +np.mean(A) +np.mean(B) +np.mean(A, axis=0) +np.mean(B, axis=0) +np.mean(A, keepdims=True) +np.mean(B, keepdims=True) +np.mean(b, out=d) +np.mean(B, out=d) + +np.std(a) +np.std(b) +np.std(c) +np.std(A) +np.std(B) +np.std(A, axis=0) +np.std(B, axis=0) +np.std(A, keepdims=True) +np.std(B, keepdims=True) +np.std(b, out=d) +np.std(B, out=d) + +np.var(a) +np.var(b) +np.var(c) +np.var(A) +np.var(B) +np.var(A, axis=0) +np.var(B, axis=0) +np.var(A, keepdims=True) +np.var(B, keepdims=True) +np.var(b, out=d) +np.var(B, out=d) diff --git a/numpy/tests/typing/reveal/fromnumeric.py b/numpy/tests/typing/reveal/fromnumeric.py index f5feb3f5f..06501f6e2 100644 --- a/numpy/tests/typing/reveal/fromnumeric.py +++ b/numpy/tests/typing/reveal/fromnumeric.py @@ -10,6 +10,7 @@ B.setflags(write=False) a = np.bool_(True) b = np.float32(1.0) c = 1.0 +d = np.array(1.0, dtype=np.float32) # writeable reveal_type(np.take(a, 0)) # E: numpy.bool_ reveal_type(np.take(b, 0)) # E: numpy.float32 @@ -203,3 +204,75 @@ reveal_type(np.amin(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] reveal_type(np.amin(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] reveal_type(np.amin(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] reveal_type(np.amin(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] + +reveal_type(np.prod(a)) # E: numpy.number +reveal_type(np.prod(b)) # E: numpy.float32 +reveal_type(np.prod(c)) # E: numpy.number +reveal_type(np.prod(A)) # E: numpy.number +reveal_type(np.prod(B)) # E: numpy.number +reveal_type(np.prod(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.prod(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.prod(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.prod(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.prod(b, out=d)) # E: numpy.ndarray +reveal_type(np.prod(B, out=d)) # E: numpy.ndarray + +reveal_type(np.cumprod(a)) # E: numpy.ndarray +reveal_type(np.cumprod(b)) # E: numpy.ndarray +reveal_type(np.cumprod(c)) # E: numpy.ndarray +reveal_type(np.cumprod(A)) # E: numpy.ndarray +reveal_type(np.cumprod(B)) # E: numpy.ndarray + +reveal_type(np.ndim(a)) # E: int +reveal_type(np.ndim(b)) # E: int +reveal_type(np.ndim(c)) # E: int +reveal_type(np.ndim(A)) # E: int +reveal_type(np.ndim(B)) # E: int + +reveal_type(np.size(a)) # E: int +reveal_type(np.size(b)) # E: int +reveal_type(np.size(c)) # E: int +reveal_type(np.size(A)) # E: int +reveal_type(np.size(B)) # E: int + +reveal_type(np.around(a)) # E: numpy.number +reveal_type(np.around(b)) # E: numpy.float32 +reveal_type(np.around(c)) # E: numpy.number +reveal_type(np.around(A)) # E: numpy.ndarray +reveal_type(np.around(B)) # E: numpy.ndarray + +reveal_type(np.mean(a)) # E: numpy.number +reveal_type(np.mean(b)) # E: numpy.number +reveal_type(np.mean(c)) # E: numpy.number +reveal_type(np.mean(A)) # E: numpy.number +reveal_type(np.mean(B)) # E: numpy.number +reveal_type(np.mean(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.mean(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.mean(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.mean(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.mean(b, out=d)) # E: numpy.ndarray +reveal_type(np.mean(B, out=d)) # E: numpy.ndarray + +reveal_type(np.std(a)) # E: numpy.number +reveal_type(np.std(b)) # E: numpy.number +reveal_type(np.std(c)) # E: numpy.number +reveal_type(np.std(A)) # E: numpy.number +reveal_type(np.std(B)) # E: numpy.number +reveal_type(np.std(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.std(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.std(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.std(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.std(b, out=d)) # E: numpy.ndarray +reveal_type(np.std(B, out=d)) # E: numpy.ndarray + +reveal_type(np.var(a)) # E: numpy.number +reveal_type(np.var(b)) # E: numpy.number +reveal_type(np.var(c)) # E: numpy.number +reveal_type(np.var(A)) # E: numpy.number +reveal_type(np.var(B)) # E: numpy.number +reveal_type(np.var(A, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.var(B, axis=0)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.var(A, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.var(B, keepdims=True)) # E: Union[numpy.number, numpy.ndarray] +reveal_type(np.var(b, out=d)) # E: numpy.ndarray +reveal_type(np.var(B, out=d)) # E: numpy.ndarray -- cgit v1.2.1 From 33b3deadf8bcd6c8aa9ac2f5594442d86fd676b1 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 24 Aug 2020 13:05:52 +0100 Subject: MAINT: Replace NpySlice_GetIndicesEx with PySlice_GetIndicesEx --- numpy/core/src/multiarray/mapping.c | 6 +++--- numpy/core/src/multiarray/nditer_pywrap.c | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index db15ff1d5..eb9125695 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -946,9 +946,9 @@ get_view_from_index(PyArrayObject *self, PyArrayObject **view, } break; case HAS_SLICE: - if (NpySlice_GetIndicesEx(indices[i].object, - PyArray_DIMS(self)[orig_dim], - &start, &stop, &step, &n_steps) < 0) { + if (PySlice_GetIndicesEx(indices[i].object, + PyArray_DIMS(self)[orig_dim], + &start, &stop, &step, &n_steps) < 0) { return -1; } if (n_steps <= 0) { diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 1c68a4803..1ea6ac15f 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -2231,8 +2231,8 @@ npyiter_subscript(NewNpyArrayIterObject *self, PyObject *op) } else if (PySlice_Check(op)) { Py_ssize_t istart = 0, iend = 0, istep = 0, islicelength; - if (NpySlice_GetIndicesEx(op, NpyIter_GetNOp(self->iter), - &istart, &iend, &istep, &islicelength) < 0) { + if (PySlice_GetIndicesEx(op, NpyIter_GetNOp(self->iter), + &istart, &iend, &istep, &islicelength) < 0) { return NULL; } if (istep != 1) { @@ -2280,8 +2280,8 @@ npyiter_ass_subscript(NewNpyArrayIterObject *self, PyObject *op, } else if (PySlice_Check(op)) { Py_ssize_t istart = 0, iend = 0, istep = 0, islicelength = 0; - if (NpySlice_GetIndicesEx(op, NpyIter_GetNOp(self->iter), - &istart, &iend, &istep, &islicelength) < 0) { + if (PySlice_GetIndicesEx(op, NpyIter_GetNOp(self->iter), + &istart, &iend, &istep, &islicelength) < 0) { return -1; } if (istep != 1) { -- cgit v1.2.1 From f854965c27d1755c918f377cc89f222d94e4e6f4 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 24 Aug 2020 13:06:31 +0100 Subject: MAINT: Replace Npy_EnterRecursiveCall with Py_EnterRecursiveCall --- numpy/core/src/multiarray/number.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index 19ac7d7f9..25a9cb396 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -826,7 +826,7 @@ _array_nonzero(PyArrayObject *mp) n = PyArray_SIZE(mp); if (n == 1) { int res; - if (Npy_EnterRecursiveCall(" while converting array to bool")) { + if (Py_EnterRecursiveCall(" while converting array to bool")) { return -1; } res = PyArray_DESCR(mp)->f->nonzero(PyArray_DATA(mp), mp); @@ -880,7 +880,7 @@ array_scalar_forward(PyArrayObject *v, /* Need to guard against recursion if our array holds references */ if (PyDataType_REFCHK(PyArray_DESCR(v))) { PyObject *res; - if (Npy_EnterRecursiveCall(where) != 0) { + if (Py_EnterRecursiveCall(where) != 0) { Py_DECREF(scalar); return NULL; } -- cgit v1.2.1 From 345da1c6fd9da8ac47f7ebb4528c5f119b2b0a47 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 24 Aug 2020 15:35:26 +0100 Subject: DOC: Remove references to PyCObject This has been replaced by PyCapsule, and no longer exists as of 3.2. --- doc/source/reference/arrays.interface.rst | 55 ++++++++++++++++++------------- numpy/core/include/numpy/ndarraytypes.h | 4 +-- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index 4e95535c0..cf4708b4b 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -199,9 +199,9 @@ array using only one attribute lookup and a well-defined C-structure. .. c:var:: __array_struct__ - A :c:type: `PyCObject` whose :c:data:`voidptr` member contains a + A :c:type:`PyCapsule` whose ``pointer`` member contains a pointer to a filled :c:type:`PyArrayInterface` structure. Memory - for the structure is dynamically created and the :c:type:`PyCObject` + for the structure is dynamically created and the :c:type:`PyCapsule` is also created with an appropriate destructor so the retriever of this attribute simply has to apply :c:func:`Py_DECREF()` to the object returned by this attribute when it is finished. Also, @@ -211,7 +211,7 @@ array using only one attribute lookup and a well-defined C-structure. must also not reallocate their memory if other objects are referencing them. -The PyArrayInterface structure is defined in ``numpy/ndarrayobject.h`` +The :c:type:`PyArrayInterface` structure is defined in ``numpy/ndarrayobject.h`` as:: typedef struct { @@ -240,13 +240,14 @@ flag is present. .. admonition:: New since June 16, 2006: - In the past most implementations used the "desc" member of the - :c:type:`PyCObject` itself (do not confuse this with the "descr" member of + In the past most implementations used the ``desc`` member of the ``PyCObject`` + (now :c:type:`PyCapsule`) itself (do not confuse this with the "descr" member of the :c:type:`PyArrayInterface` structure above --- they are two separate things) to hold the pointer to the object exposing the interface. - This is now an explicit part of the interface. Be sure to own a - reference to the object when the :c:type:`PyCObject` is created using - :c:type:`PyCObject_FromVoidPtrAndDesc`. + This is now an explicit part of the interface. Be sure to take a + reference to the object and call :c:func:`PyCapsule_SetContext` before + returning the :c:type:`PyCapsule`, and configure a destructor to decref this + reference. Type description examples @@ -315,25 +316,35 @@ largely aesthetic. In particular: 1. The PyArrayInterface structure had no descr member at the end (and therefore no flag ARR_HAS_DESCR) -2. The desc member of the PyCObject returned from __array_struct__ was +2. The ``context`` member of the :c:type:`PyCapsule` (formally the ``desc`` + member of the ``PyCObject``) returned from ``__array_struct__`` was not specified. Usually, it was the object exposing the array (so that a reference to it could be kept and destroyed when the - C-object was destroyed). Now it must be a tuple whose first - element is a string with "PyArrayInterface Version #" and whose - second element is the object exposing the array. + C-object was destroyed). It is now an explicit requirement that this field + be used in some way to hold a reference to the owning object. -3. The tuple returned from __array_interface__['data'] used to be a + .. note:: + + Until August 2020, this said: + + Now it must be a tuple whose first element is a string with "PyArrayInterface Version #" and whose + second element is the object exposing the array. + + This design was retracted in , and is an error to have relied + upon in the intermediate 14 years. + +3. The tuple returned from ``__array_interface__['data']`` used to be a hex-string (now it is an integer or a long integer). -4. There was no __array_interface__ attribute instead all of the keys - (except for version) in the __array_interface__ dictionary were +4. There was no ``__array_interface__`` attribute instead all of the keys + (except for version) in the ``__array_interface__`` dictionary were their own attribute: Thus to obtain the Python-side information you had to access separately the attributes: - * __array_data__ - * __array_shape__ - * __array_strides__ - * __array_typestr__ - * __array_descr__ - * __array_offset__ - * __array_mask__ + * ``__array_data__`` + * ``__array_shape__`` + * ``__array_strides__`` + * ``__array_typestr__`` + * ``__array_descr__`` + * ``__array_offset__`` + * ``__array_mask__`` diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index bbcf468c1..6eca4afdb 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1759,8 +1759,8 @@ typedef struct { } npy_stride_sort_item; /************************************************************ - * This is the form of the struct that's returned pointed by the - * PyCObject attribute of an array __array_struct__. See + * This is the form of the struct that's stored in the + * PyCapsule returned by an array's __array_struct__ attribute. See * https://docs.scipy.org/doc/numpy/reference/arrays.interface.html for the full * documentation. ************************************************************/ -- cgit v1.2.1 From ed2cd346034d3af49a364bce5acaee719d88379f Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 25 Aug 2020 14:46:02 +0100 Subject: MAINT: Remove references to PyStringObject --- numpy/core/include/numpy/arrayscalars.h | 3 +-- numpy/core/src/multiarray/scalarapi.c | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/numpy/core/include/numpy/arrayscalars.h b/numpy/core/include/numpy/arrayscalars.h index 6dce88df3..b282a2cd4 100644 --- a/numpy/core/include/numpy/arrayscalars.h +++ b/numpy/core/include/numpy/arrayscalars.h @@ -134,8 +134,7 @@ typedef struct { char obval; } PyScalarObject; -#define PyStringScalarObject PyStringObject -#define PyStringScalarObject PyStringObject +#define PyStringScalarObject PyBytesObject typedef struct { /* note that the PyObject_HEAD macro lives right here */ PyUnicodeObject base; diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index 4dee259f8..bcd777183 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -755,8 +755,8 @@ PyArray_Scalar(void *data, PyArray_Descr *descr, PyObject *base) } if (PyTypeNum_ISFLEXIBLE(type_num)) { if (type_num == NPY_STRING) { - destptr = PyString_AS_STRING(obj); - ((PyStringObject *)obj)->ob_shash = -1; + destptr = PyBytes_AS_STRING(obj); + ((PyBytesObject *)obj)->ob_shash = -1; memcpy(destptr, data, itemsize); return obj; } -- cgit v1.2.1 From 9f29c08a5fcc097cd157be7be82f7114da60e23b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 25 Aug 2020 18:45:00 -0500 Subject: BUG: Remove Void special case for "safe casting" There were some (dubious) special cases for "safe casting" which was only available on the C-side (np.can_cast does not hit this code). These special cases are almost only active for user dtypes and seem generally incorrect and unnecessary. Without this change, the test will fail due to promoting rational to the void dtype (whichever that is). --- numpy/core/src/multiarray/convert_datatype.c | 19 ------------------- numpy/core/tests/test_numeric.py | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 1f5845eb7..d9121707b 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -343,25 +343,6 @@ PyArray_CanCastSafely(int fromtype, int totype) if (fromtype == totype) { return 1; } - /* Special-cases for some types */ - switch (fromtype) { - case NPY_DATETIME: - case NPY_TIMEDELTA: - case NPY_OBJECT: - case NPY_VOID: - return 0; - case NPY_BOOL: - return 1; - } - switch (totype) { - case NPY_BOOL: - case NPY_DATETIME: - case NPY_TIMEDELTA: - return 0; - case NPY_OBJECT: - case NPY_VOID: - return 1; - } from = PyArray_DescrFromType(fromtype); /* diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index badf48b33..0b27c54dd 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -14,6 +14,7 @@ from numpy.testing import ( assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_warns, assert_array_max_ulp, HAS_REFCOUNT ) +from numpy.core._rational_tests import rational from hypothesis import assume, given, strategies as st from hypothesis.extra import numpy as hynp @@ -863,6 +864,30 @@ class TestTypes: assert_equal(np.promote_types('m8', '>m8'), np.dtype('m8')) + def test_can_cast_and_promote_usertypes(self): + # The rational type defines safe casting for signed integers, + # boolean. Rational itself *does* cast safely to double. + # (rational does not actually cast to all signed integers, e.g. + # int64 can be both long and longlong and it registers only the first) + valid_types = ["int8", "int16", "int32", "int64", "bool"] + invalid_types = "BHILQP" + "FDG" + "mM" + "f" + "V" + + rational_dt = np.dtype(rational) + for numpy_dtype in valid_types: + numpy_dtype = np.dtype(numpy_dtype) + assert np.can_cast(numpy_dtype, rational_dt) + assert np.promote_types(numpy_dtype, rational_dt) is rational_dt + + for numpy_dtype in invalid_types: + numpy_dtype = np.dtype(numpy_dtype) + assert not np.can_cast(numpy_dtype, rational_dt) + with pytest.raises(TypeError): + np.promote_types(numpy_dtype, rational_dt) + + double_dt = np.dtype("double") + assert np.can_cast(rational_dt, double_dt) + assert np.promote_types(double_dt, rational_dt) is double_dt + def test_promote_types_strings(self): assert_equal(np.promote_types('bool', 'S'), np.dtype('S5')) assert_equal(np.promote_types('b', 'S'), np.dtype('S4')) -- cgit v1.2.1 From fe04c2aed76b068743ff4ad3c9ed01feb871cef2 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 26 Aug 2020 09:36:47 +0100 Subject: Improve wording of note --- doc/source/reference/arrays.interface.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index cf4708b4b..c6da326d7 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -330,8 +330,10 @@ largely aesthetic. In particular: Now it must be a tuple whose first element is a string with "PyArrayInterface Version #" and whose second element is the object exposing the array. - This design was retracted in , and is an error to have relied - upon in the intermediate 14 years. + This design was retracted almost immediately after it was proposed, in + . + Despite 14 years of documentation to the contrary, at no point was it valid + to assume that ``__array_interface__`` capsules held this tuple content. 3. The tuple returned from ``__array_interface__['data']`` used to be a hex-string (now it is an integer or a long integer). -- cgit v1.2.1 From ada91e01ab0b44bd7af71c404c8b7020b2dce2bc Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 26 Aug 2020 09:38:22 +0100 Subject: Wrap lines --- doc/source/reference/arrays.interface.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/source/reference/arrays.interface.rst b/doc/source/reference/arrays.interface.rst index c6da326d7..73e4aef0c 100644 --- a/doc/source/reference/arrays.interface.rst +++ b/doc/source/reference/arrays.interface.rst @@ -327,13 +327,15 @@ largely aesthetic. In particular: Until August 2020, this said: - Now it must be a tuple whose first element is a string with "PyArrayInterface Version #" and whose - second element is the object exposing the array. + Now it must be a tuple whose first element is a string with + "PyArrayInterface Version #" and whose second element is the object + exposing the array. This design was retracted almost immediately after it was proposed, in . - Despite 14 years of documentation to the contrary, at no point was it valid - to assume that ``__array_interface__`` capsules held this tuple content. + Despite 14 years of documentation to the contrary, at no point was it + valid to assume that ``__array_interface__`` capsules held this tuple + content. 3. The tuple returned from ``__array_interface__['data']`` used to be a hex-string (now it is an integer or a long integer). -- cgit v1.2.1 From a4aeafef357f803e96af79abcf158d6d776161f7 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Wed, 26 Aug 2020 10:00:32 +0100 Subject: MAINT: Remove NPY_COPY_PYOBJECT_PTR This macro was made useless in d6278bdc47d12acc5d3edfad3ad0654055f77737, this is just a final cleanup. This allows a header to be removed from `common.h` --- numpy/core/include/numpy/npy_cpu.h | 3 --- numpy/core/src/multiarray/arraytypes.c.src | 20 ++++++++++---------- numpy/core/src/multiarray/common.h | 1 - numpy/core/src/multiarray/dtype_transfer.c | 21 ++++++++++----------- numpy/core/src/multiarray/methods.c | 6 +++--- numpy/core/src/multiarray/refcount.c | 14 +++++++------- numpy/core/src/multiarray/shape.c | 2 +- numpy/core/src/npymath/npy_math_private.h | 1 - 8 files changed, 31 insertions(+), 37 deletions(-) diff --git a/numpy/core/include/numpy/npy_cpu.h b/numpy/core/include/numpy/npy_cpu.h index 509e23a51..4dbf9d84e 100644 --- a/numpy/core/include/numpy/npy_cpu.h +++ b/numpy/core/include/numpy/npy_cpu.h @@ -24,7 +24,6 @@ #define _NPY_CPUARCH_H_ #include "numpyconfig.h" -#include /* for memcpy */ #if defined( __i386__ ) || defined(i386) || defined(_M_IX86) /* @@ -111,8 +110,6 @@ information about your platform (OS, CPU and compiler) #endif -#define NPY_COPY_PYOBJECT_PTR(dst, src) memcpy(dst, src, sizeof(PyObject *)) - #if (defined(NPY_CPU_X86) || defined(NPY_CPU_AMD64)) #define NPY_CPU_HAVE_UNALIGNED_ACCESS 1 #else diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 3fee587b9..4767901ef 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -648,7 +648,7 @@ static PyObject * OBJECT_getitem(void *ip, void *NPY_UNUSED(ap)) { PyObject *obj; - NPY_COPY_PYOBJECT_PTR(&obj, ip); + memcpy(&obj, ip, sizeof(obj)); if (obj == NULL) { Py_RETURN_NONE; } @@ -664,12 +664,12 @@ OBJECT_setitem(PyObject *op, void *ov, void *NPY_UNUSED(ap)) { PyObject *obj; - NPY_COPY_PYOBJECT_PTR(&obj, ov); + memcpy(&obj, ov, sizeof(obj)); Py_INCREF(op); Py_XDECREF(obj); - NPY_COPY_PYOBJECT_PTR(ov, &op); + memcpy(ov, &op, sizeof(op)); return PyErr_Occurred() ? -1 : 0; } @@ -2237,11 +2237,11 @@ OBJECT_copyswapn(PyObject **dst, npy_intp dstride, PyObject **src, dstp = (unsigned char*)dst; srcp = (unsigned char*)src; for (i = 0; i < n; i++) { - NPY_COPY_PYOBJECT_PTR(&tmp, srcp); + memcpy(&tmp, srcp, sizeof(tmp)); Py_XINCREF(tmp); - NPY_COPY_PYOBJECT_PTR(&tmp, dstp); + memcpy(&tmp, dstp, sizeof(tmp)); Py_XDECREF(tmp); - NPY_COPY_PYOBJECT_PTR(dstp, srcp); + memcpy(dstp, srcp, sizeof(tmp)); dstp += dstride; srcp += sstride; } @@ -2265,11 +2265,11 @@ OBJECT_copyswap(PyObject **dst, PyObject **src, int NPY_UNUSED(swap), } else { PyObject *tmp; - NPY_COPY_PYOBJECT_PTR(&tmp, src); + memcpy(&tmp, src, sizeof(tmp)); Py_XINCREF(tmp); - NPY_COPY_PYOBJECT_PTR(&tmp, dst); + memcpy(&tmp, dst, sizeof(tmp)); Py_XDECREF(tmp); - NPY_COPY_PYOBJECT_PTR(dst, src); + memcpy(dst, src, sizeof(tmp)); } } } @@ -2686,7 +2686,7 @@ OBJECT_nonzero (PyObject **ip, PyArrayObject *ap) } else { PyObject *obj; - NPY_COPY_PYOBJECT_PTR(&obj, ip); + memcpy(&obj, ip, sizeof(obj)); if (obj == NULL) { return NPY_FALSE; } diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index 4410825fa..ef9bc79da 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -2,7 +2,6 @@ #define _NPY_PRIVATE_COMMON_H_ #include "structmember.h" #include -#include #include #include #include "npy_import.h" diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index c9868a2c8..42c66ee7f 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -17,7 +17,6 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION #define _MULTIARRAYMODULE #include -#include #include "npy_pycompat.h" @@ -114,18 +113,18 @@ _strided_to_strided_move_references(char *dst, npy_intp dst_stride, { PyObject *src_ref = NULL, *dst_ref = NULL; while (N > 0) { - NPY_COPY_PYOBJECT_PTR(&src_ref, src); - NPY_COPY_PYOBJECT_PTR(&dst_ref, dst); + memcpy(&src_ref, src, sizeof(src_ref)); + memcpy(&dst_ref, dst, sizeof(dst_ref)); /* Release the reference in dst */ NPY_DT_DBG_REFTRACE("dec dst ref", dst_ref); Py_XDECREF(dst_ref); /* Move the reference */ NPY_DT_DBG_REFTRACE("move src ref", src_ref); - NPY_COPY_PYOBJECT_PTR(dst, &src_ref); + memcpy(dst, &src_ref, sizeof(src_ref)); /* Set the source reference to NULL */ src_ref = NULL; - NPY_COPY_PYOBJECT_PTR(src, &src_ref); + memcpy(src, &src_ref, sizeof(src_ref)); src += src_stride; dst += dst_stride; @@ -143,12 +142,12 @@ _strided_to_strided_copy_references(char *dst, npy_intp dst_stride, { PyObject *src_ref = NULL, *dst_ref = NULL; while (N > 0) { - NPY_COPY_PYOBJECT_PTR(&src_ref, src); - NPY_COPY_PYOBJECT_PTR(&dst_ref, dst); + memcpy(&src_ref, src, sizeof(src_ref)); + memcpy(&dst_ref, dst, sizeof(dst_ref)); /* Copy the reference */ NPY_DT_DBG_REFTRACE("copy src ref", src_ref); - NPY_COPY_PYOBJECT_PTR(dst, &src_ref); + memcpy(dst, &src_ref, sizeof(src_ref)); /* Claim the reference */ Py_XINCREF(src_ref); /* Release the reference in dst */ @@ -694,7 +693,7 @@ _aligned_strided_to_strided_cast_decref_src(char *dst, npy_intp dst_stride, return -1; } /* After casting, decrement the source ref and set it to NULL */ - NPY_COPY_PYOBJECT_PTR(&src_ref, src); + memcpy(&src_ref, src, sizeof(src_ref)); Py_XDECREF(src_ref); memset(src, 0, sizeof(PyObject *)); NPY_DT_DBG_REFTRACE("dec src ref (cast object -> not object)", src_ref); @@ -3218,7 +3217,7 @@ _null_to_strided_reference_setzero(char *dst, PyObject *dst_ref = NULL; while (N > 0) { - NPY_COPY_PYOBJECT_PTR(&dst_ref, dst); + memcpy(&dst_ref, dst, sizeof(dst_ref)); /* Release the reference in dst and set it to NULL */ NPY_DT_DBG_REFTRACE("dec dest ref (to set zero)", dst_ref); @@ -3349,7 +3348,7 @@ _strided_to_null_dec_src_ref_reference(char *NPY_UNUSED(dst), while (N > 0) { /* Release the reference in src and set it to NULL */ NPY_DT_DBG_REFTRACE("dec src ref (null dst)", src_ref); - NPY_COPY_PYOBJECT_PTR(&src_ref, src); + memcpy(&src_ref, src, sizeof(src_ref)); Py_XDECREF(src_ref); memset(src, 0, sizeof(PyObject *)); diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index ae2dceb10..d2e8ba06e 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1508,14 +1508,14 @@ _deepcopy_call(char *iptr, char *optr, PyArray_Descr *dtype, else { PyObject *itemp, *otemp; PyObject *res; - NPY_COPY_PYOBJECT_PTR(&itemp, iptr); - NPY_COPY_PYOBJECT_PTR(&otemp, optr); + memcpy(&itemp, iptr, sizeof(itemp)); + memcpy(&otemp, optr, sizeof(otemp)); Py_XINCREF(itemp); /* call deepcopy on this argument */ res = PyObject_CallFunctionObjArgs(deepcopy, itemp, visit, NULL); Py_XDECREF(itemp); Py_XDECREF(otemp); - NPY_COPY_PYOBJECT_PTR(optr, &res); + memcpy(optr, &res, sizeof(res)); } } diff --git a/numpy/core/src/multiarray/refcount.c b/numpy/core/src/multiarray/refcount.c index 985d187fa..0f84449af 100644 --- a/numpy/core/src/multiarray/refcount.c +++ b/numpy/core/src/multiarray/refcount.c @@ -36,7 +36,7 @@ PyArray_Item_INCREF(char *data, PyArray_Descr *descr) return; } if (descr->type_num == NPY_OBJECT) { - NPY_COPY_PYOBJECT_PTR(&temp, data); + memcpy(&temp, data, sizeof(temp)); Py_XINCREF(temp); } else if (PyDataType_HASFIELDS(descr)) { @@ -98,7 +98,7 @@ PyArray_Item_XDECREF(char *data, PyArray_Descr *descr) } if (descr->type_num == NPY_OBJECT) { - NPY_COPY_PYOBJECT_PTR(&temp, data); + memcpy(&temp, data, sizeof(temp)); Py_XDECREF(temp); } else if (PyDataType_HASFIELDS(descr)) { @@ -181,7 +181,7 @@ PyArray_INCREF(PyArrayObject *mp) } else { for( i = 0; i < n; i++, data++) { - NPY_COPY_PYOBJECT_PTR(&temp, data); + memcpy(&temp, data, sizeof(temp)); Py_XINCREF(temp); } } @@ -192,7 +192,7 @@ PyArray_INCREF(PyArrayObject *mp) return -1; } while(it->index < it->size) { - NPY_COPY_PYOBJECT_PTR(&temp, it->dataptr); + memcpy(&temp, it->dataptr, sizeof(temp)); Py_XINCREF(temp); PyArray_ITER_NEXT(it); } @@ -238,7 +238,7 @@ PyArray_XDECREF(PyArrayObject *mp) } else { for (i = 0; i < n; i++, data++) { - NPY_COPY_PYOBJECT_PTR(&temp, data); + memcpy(&temp, data, sizeof(temp)); Py_XDECREF(temp); } } @@ -246,7 +246,7 @@ PyArray_XDECREF(PyArrayObject *mp) else { /* handles misaligned data too */ PyArray_RawIterBaseInit(&it, mp); while(it.index < it.size) { - NPY_COPY_PYOBJECT_PTR(&temp, it.dataptr); + memcpy(&temp, it.dataptr, sizeof(temp)); Py_XDECREF(temp); PyArray_ITER_NEXT(&it); } @@ -309,7 +309,7 @@ _fillobject(char *optr, PyObject *obj, PyArray_Descr *dtype) } if (dtype->type_num == NPY_OBJECT) { Py_XINCREF(obj); - NPY_COPY_PYOBJECT_PTR(optr, &obj); + memcpy(optr, &obj, sizeof(obj)); } else if (PyDataType_HASFIELDS(dtype)) { PyObject *key, *value, *title = NULL; diff --git a/numpy/core/src/multiarray/shape.c b/numpy/core/src/multiarray/shape.c index 397d539c1..1a38fe956 100644 --- a/numpy/core/src/multiarray/shape.c +++ b/numpy/core/src/multiarray/shape.c @@ -332,7 +332,7 @@ _putzero(char *optr, PyObject *zero, PyArray_Descr *dtype) for (i = 0; i < nsize; i++) { Py_INCREF(zero); - NPY_COPY_PYOBJECT_PTR(optr, &zero); + memcpy(optr, &zero, sizeof(zero)); optr += sizeof(zero); } } diff --git a/numpy/core/src/npymath/npy_math_private.h b/numpy/core/src/npymath/npy_math_private.h index e4a919db6..212d11a0b 100644 --- a/numpy/core/src/npymath/npy_math_private.h +++ b/numpy/core/src/npymath/npy_math_private.h @@ -25,7 +25,6 @@ #include "npy_fpmath.h" #include "numpy/npy_math.h" -#include "numpy/npy_cpu.h" #include "numpy/npy_endian.h" #include "numpy/npy_common.h" -- cgit v1.2.1 From 5281f86f386e530b10131193f3a7d5f338d8daaa Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Fri, 21 Aug 2020 17:53:46 +0200 Subject: BLD: Check for reduce intrinsics and AVX512BW mask operations - Extending Distutils::CompilerOpt to allow adding extra separate test cases related to a certain CPU feature without affecting its availability. - Add test cases for reduce intrinsics and AVX512BW mask operations, they can be reached through C #defentions NPY_HAVE_AVX512BW_MASK and NPY_HAVE_AVX512F_REDUCE. --- numpy/distutils/ccompiler_opt.py | 92 +++++++++++++++++++++--- numpy/distutils/checks/extra_avx512bw_mask.c | 18 +++++ numpy/distutils/checks/extra_avx512f_reduce.c | 41 +++++++++++ numpy/distutils/tests/test_ccompiler_opt_conf.py | 23 +++++- 4 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 numpy/distutils/checks/extra_avx512bw_mask.c create mode 100644 numpy/distutils/checks/extra_avx512f_reduce.c diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py index 85dc2f1e8..46f04af47 100644 --- a/numpy/distutils/ccompiler_opt.py +++ b/numpy/distutils/ccompiler_opt.py @@ -152,6 +152,18 @@ class _Config: By default(None), treated as True if the feature contains at least one applicable flag. see `feature_can_autovec()` + "extra_checks": str or list, optional + Extra test cases names for the CPU feature that need to be tested + against the compiler. + + Each test case name must has a C file located at 'conf_check_path' + contains at least one intrinsic or function related to the test case + named extra_xxxx.c where xxxx is the case name in lower case. + + If the compiler able to successfully compile the C file then `CCompilerOpt` + will add C #definition for it into the main dispatch header, e.g. + #define {conf_c_prefix}_XXXX where XXXX is the case name in upper case. + **NOTES**: * space can be used as separator with options that supports "str or list" * case-sensitive for all values and feature name must be in upper-case. @@ -230,7 +242,10 @@ class _Config: F16C = dict(interest=11, implies="AVX"), FMA3 = dict(interest=12, implies="F16C"), AVX2 = dict(interest=13, implies="F16C"), - AVX512F = dict(interest=20, implies="FMA3 AVX2", implies_detect=False), + AVX512F = dict( + interest=20, implies="FMA3 AVX2", implies_detect=False, + extra_checks="AVX512F_REDUCE" + ), AVX512CD = dict(interest=21, implies="AVX512F"), AVX512_KNL = dict( interest=40, implies="AVX512CD", group="AVX512ER AVX512PF", @@ -243,7 +258,8 @@ class _Config: ), AVX512_SKX = dict( interest=42, implies="AVX512CD", group="AVX512VL AVX512BW AVX512DQ", - detect="AVX512_SKX", implies_detect=False + detect="AVX512_SKX", implies_detect=False, + extra_checks="AVX512BW_MASK" ), AVX512_CLX = dict( interest=43, implies="AVX512_SKX", group="AVX512VNNI", @@ -673,7 +689,7 @@ class _Distutils: # intel and msvc compilers don't raise # fatal errors when flags are wrong or unsupported ".*(" - "warning D9002|" # msvc, it should be work with any language. + "warning D9002|" # msvc, it should be work with any language. "invalid argument for option" # intel ").*" ) @@ -1137,7 +1153,7 @@ class _Feature: continue # list is used internally for these options for option in ( - "implies", "group", "detect", "headers", "flags" + "implies", "group", "detect", "headers", "flags", "extra_checks" ) : oval = feature.get(option) if isinstance(oval, str): @@ -1439,7 +1455,7 @@ class _Feature: self.conf_check_path, "cpu_%s.c" % name.lower() ) if not os.path.exists(test_path): - self.dist_fatal("feature test file is not exist", path) + self.dist_fatal("feature test file is not exist", test_path) test = self.dist_test(test_path, force_flags + self.cc_flags["werror"]) if not test: @@ -1487,6 +1503,45 @@ class _Feature: can = valid_flags and any(valid_flags) return can + @_Cache.me + def feature_extra_checks(self, name): + """ + Return a list of supported extra checks after testing them against + the compiler. + + Parameters + ---------- + names: str + CPU feature name in uppercase. + """ + assert isinstance(name, str) + d = self.feature_supported[name] + extra_checks = d.get("extra_checks", []) + if not extra_checks: + return [] + + self.dist_log("Testing extra checks for feature '%s'" % name, extra_checks) + flags = self.feature_flags(name) + available = [] + not_available = [] + for chk in extra_checks: + test_path = os.path.join( + self.conf_check_path, "extra_%s.c" % chk.lower() + ) + if not os.path.exists(test_path): + self.dist_fatal("extra check file is not exist", test_path) + + is_supported = self.dist_test(test_path, flags + self.cc_flags["werror"]) + if is_supported: + available.append(chk) + else: + not_available.append(chk) + + if not_available: + self.dist_log("testing failed for checks", not_available, stderr=True) + return available + + def feature_c_preprocessor(self, feature_name, tabs=0): """ Generate C preprocessor definitions and include headers of a CPU feature. @@ -1520,14 +1575,18 @@ class _Feature: prepr += [ "#include <%s>" % h for h in feature.get("headers", []) ] - group = feature.get("group", []) - for f in group: - # Guard features in case of duplicate definitions + + extra_defs = feature.get("group", []) + extra_defs += self.feature_extra_checks(feature_name) + for edef in extra_defs: + # Guard extra definitions in case of duplicate with + # another feature prepr += [ - "#ifndef %sHAVE_%s" % (self.conf_c_prefix, f), - "\t#define %sHAVE_%s 1" % (self.conf_c_prefix, f), + "#ifndef %sHAVE_%s" % (self.conf_c_prefix, edef), + "\t#define %sHAVE_%s 1" % (self.conf_c_prefix, edef), "#endif", ] + if tabs > 0: prepr = [('\t'*tabs) + l for l in prepr] return '\n'.join(prepr) @@ -2269,6 +2328,12 @@ class CCompilerOpt(_Config, _Distutils, _Cache, _CCompiler, _Feature, _Parse): baseline_rows.append(( "Flags", (' '.join(baseline_flags) if baseline_flags else "none") )) + extra_checks = [] + for name in baseline_names: + extra_checks += self.feature_extra_checks(name) + baseline_rows.append(( + "Extra checks", (' '.join(extra_checks) if extra_checks else "none") + )) ########## dispatch ########## if self.cc_noopt: @@ -2307,14 +2372,21 @@ class CCompilerOpt(_Config, _Distutils, _Cache, _CCompiler, _Feature, _Parse): else: dispatch_rows.append(("Generated", '')) for tar in self.feature_sorted(target_sources): + tar_as_seq = [tar] if isinstance(tar, str) else tar sources = target_sources[tar] name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar) flags = ' '.join(self.feature_flags(tar)) implies = ' '.join(self.feature_sorted(self.feature_implies(tar))) detect = ' '.join(self.feature_detect(tar)) + extra_checks = [] + for name in tar_as_seq: + extra_checks += self.feature_extra_checks(name) + extra_checks = (' '.join(extra_checks) if extra_checks else "none") + dispatch_rows.append(('', '')) dispatch_rows.append((name, implies)) dispatch_rows.append(("Flags", flags)) + dispatch_rows.append(("Extra checks", extra_checks)) dispatch_rows.append(("Detect", detect)) for src in sources: dispatch_rows.append(("", src)) diff --git a/numpy/distutils/checks/extra_avx512bw_mask.c b/numpy/distutils/checks/extra_avx512bw_mask.c new file mode 100644 index 000000000..9cfd0c2a5 --- /dev/null +++ b/numpy/distutils/checks/extra_avx512bw_mask.c @@ -0,0 +1,18 @@ +#include +/** + * Test BW mask operations due to: + * - MSVC has supported it since vs2019 see, + * https://developercommunity.visualstudio.com/content/problem/518298/missing-avx512bw-mask-intrinsics.html + * - Clang >= v8.0 + * - GCC >= v7.1 + */ +int main(void) +{ + __mmask64 m64 = _mm512_cmpeq_epi8_mask(_mm512_set1_epi8((char)1), _mm512_set1_epi8((char)1)); + m64 = _kor_mask64(m64, m64); + m64 = _kxor_mask64(m64, m64); + m64 = _cvtu64_mask64(_cvtmask64_u64(m64)); + m64 = _mm512_kunpackd(m64, m64); + m64 = (__mmask64)_mm512_kunpackw((__mmask32)m64, (__mmask32)m64); + return (int)_cvtmask64_u64(m64); +} diff --git a/numpy/distutils/checks/extra_avx512f_reduce.c b/numpy/distutils/checks/extra_avx512f_reduce.c new file mode 100644 index 000000000..f979d504e --- /dev/null +++ b/numpy/distutils/checks/extra_avx512f_reduce.c @@ -0,0 +1,41 @@ +#include +/** + * The following intrinsics don't have direct native support but compilers + * tend to emulate them. + * They're usually supported by gcc >= 7.1, clang >= 4 and icc >= 19 + */ +int main(void) +{ + __m512 one_ps = _mm512_set1_ps(1.0f); + __m512d one_pd = _mm512_set1_pd(1.0); + __m512i one_i64 = _mm512_set1_epi64(1.0); + // add + float sum_ps = _mm512_reduce_add_ps(one_ps); + double sum_pd = _mm512_reduce_add_pd(one_pd); + int sum_int = (int)_mm512_reduce_add_epi64(one_i64); + sum_int += (int)_mm512_reduce_add_epi32(one_i64); + // mul + sum_ps += _mm512_reduce_mul_ps(one_ps); + sum_pd += _mm512_reduce_mul_pd(one_pd); + sum_int += (int)_mm512_reduce_mul_epi64(one_i64); + sum_int += (int)_mm512_reduce_mul_epi32(one_i64); + // min + sum_ps += _mm512_reduce_min_ps(one_ps); + sum_pd += _mm512_reduce_min_pd(one_pd); + sum_int += (int)_mm512_reduce_min_epi32(one_i64); + sum_int += (int)_mm512_reduce_min_epu32(one_i64); + sum_int += (int)_mm512_reduce_min_epi64(one_i64); + // max + sum_ps += _mm512_reduce_max_ps(one_ps); + sum_pd += _mm512_reduce_max_pd(one_pd); + sum_int += (int)_mm512_reduce_max_epi32(one_i64); + sum_int += (int)_mm512_reduce_max_epu32(one_i64); + sum_int += (int)_mm512_reduce_max_epi64(one_i64); + // and + sum_int += (int)_mm512_reduce_and_epi32(one_i64); + sum_int += (int)_mm512_reduce_and_epi64(one_i64); + // or + sum_int += (int)_mm512_reduce_or_epi32(one_i64); + sum_int += (int)_mm512_reduce_or_epi64(one_i64); + return (int)sum_ps + (int)sum_pd + sum_int; +} diff --git a/numpy/distutils/tests/test_ccompiler_opt_conf.py b/numpy/distutils/tests/test_ccompiler_opt_conf.py index 2f83a59e0..5c4717192 100644 --- a/numpy/distutils/tests/test_ccompiler_opt_conf.py +++ b/numpy/distutils/tests/test_ccompiler_opt_conf.py @@ -66,11 +66,12 @@ class _TestConfFeatures(FakeCCompilerOpt): self.test_implies(error_msg, search_in, feature_name, feature_dict) self.test_group(error_msg, search_in, feature_name, feature_dict) + self.test_extra_checks(error_msg, search_in, feature_name, feature_dict) def test_option_types(self, error_msg, option, val): for tp, available in ( ((str, list), ( - "implies", "headers", "flags", "group", "detect" + "implies", "headers", "flags", "group", "detect", "extra_checks" )), ((str,), ("disable",)), ((int,), ("interest",)), @@ -96,7 +97,7 @@ class _TestConfFeatures(FakeCCompilerOpt): def test_duplicates(self, error_msg, option, val): if option not in ( - "implies", "headers", "flags", "group", "detect" + "implies", "headers", "flags", "group", "detect", "extra_checks" ) : return if isinstance(val, str): @@ -151,6 +152,24 @@ class _TestConfFeatures(FakeCCompilerOpt): option, f )) + def test_extra_checks(self, error_msg, search_in, feature_name, feature_dict): + if feature_dict.get("disabled") is not None: + return + extra_checks = feature_dict.get("extra_checks", "") + if not extra_checks: + return + if isinstance(extra_checks, str): + extra_checks = extra_checks.split() + + for f in extra_checks: + impl_dict = search_in.get(f) + if not impl_dict or "disable" in impl_dict: + continue + raise AssertionError(error_msg + \ + "in option '%s', extra test case '%s' already exists as a feature name" % ( + option, f + )) + class TestConfFeatures(unittest.TestCase): def __init__(self, methodName="runTest"): unittest.TestCase.__init__(self, methodName) -- cgit v1.2.1 From 5789103b2b39fc7d26852654ba283365a6ea8314 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 26 Aug 2020 13:12:41 +0200 Subject: TST: remove unnecessary backslashes from CCompilerOpt config test Also fix assertion error msg of group/extra_checks Co-authored-by: Eric Wieser --- numpy/distutils/tests/test_ccompiler_opt_conf.py | 36 ++++++++---------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/numpy/distutils/tests/test_ccompiler_opt_conf.py b/numpy/distutils/tests/test_ccompiler_opt_conf.py index 5c4717192..244748e58 100644 --- a/numpy/distutils/tests/test_ccompiler_opt_conf.py +++ b/numpy/distutils/tests/test_ccompiler_opt_conf.py @@ -84,16 +84,14 @@ class _TestConfFeatures(FakeCCompilerOpt): if not isinstance(val, tp): error_tp = [t.__name__ for t in (*tp,)] error_tp = ' or '.join(error_tp) - raise AssertionError(error_msg + \ + raise AssertionError(error_msg + "expected '%s' type for option '%s' not '%s'" % ( error_tp, option, type(val).__name__ )) break if not found_it: - raise AssertionError(error_msg + \ - "invalid option name '%s'" % option - ) + raise AssertionError(error_msg + "invalid option name '%s'" % option) def test_duplicates(self, error_msg, option, val): if option not in ( @@ -104,9 +102,7 @@ class _TestConfFeatures(FakeCCompilerOpt): val = val.split() if len(val) != len(set(val)): - raise AssertionError(error_msg + \ - "duplicated values in option '%s'" % option - ) + raise AssertionError(error_msg + "duplicated values in option '%s'" % option) def test_implies(self, error_msg, search_in, feature_name, feature_dict): if feature_dict.get("disabled") is not None: @@ -118,21 +114,15 @@ class _TestConfFeatures(FakeCCompilerOpt): implies = implies.split() if feature_name in implies: - raise AssertionError(error_msg + \ - "feature implies itself" - ) + raise AssertionError(error_msg + "feature implies itself") for impl in implies: impl_dict = search_in.get(impl) if impl_dict is not None: if "disable" in impl_dict: - raise AssertionError(error_msg + \ - "implies disabled feature '%s'" % impl - ) + raise AssertionError(error_msg + "implies disabled feature '%s'" % impl) continue - raise AssertionError(error_msg + \ - "implies non-exist feature '%s'" % impl - ) + raise AssertionError(error_msg + "implies non-exist feature '%s'" % impl) def test_group(self, error_msg, search_in, feature_name, feature_dict): if feature_dict.get("disabled") is not None: @@ -147,10 +137,9 @@ class _TestConfFeatures(FakeCCompilerOpt): impl_dict = search_in.get(f) if not impl_dict or "disable" in impl_dict: continue - raise AssertionError(error_msg + \ - "in option '%s', '%s' already exists as a feature name" % ( - option, f - )) + raise AssertionError(error_msg + + "in option 'group', '%s' already exists as a feature name" % f + ) def test_extra_checks(self, error_msg, search_in, feature_name, feature_dict): if feature_dict.get("disabled") is not None: @@ -165,10 +154,9 @@ class _TestConfFeatures(FakeCCompilerOpt): impl_dict = search_in.get(f) if not impl_dict or "disable" in impl_dict: continue - raise AssertionError(error_msg + \ - "in option '%s', extra test case '%s' already exists as a feature name" % ( - option, f - )) + raise AssertionError(error_msg + + "in option 'extra_checks', extra test case '%s' already exists as a feature name" % f + ) class TestConfFeatures(unittest.TestCase): def __init__(self, methodName="runTest"): -- cgit v1.2.1 From a68a3ada494d9f66625c7ae94d12635021992e3a Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 25 Aug 2020 14:57:20 +0100 Subject: BUG: Do not throw UnicodeError for non-ascii typestrs in __array_interface__ Instead, just throw ValueError like `np.dtype` does This also fixes unsafe error handling of a call to PyArray_DescrConverter2 --- numpy/core/src/multiarray/ctors.c | 67 +++++++++++++------------------------ numpy/core/tests/test_multiarray.py | 12 +++++++ 2 files changed, 35 insertions(+), 44 deletions(-) diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 6add032bf..c9b1614f4 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1809,37 +1809,19 @@ PyArray_FromStructInterface(PyObject *input) */ NPY_NO_EXPORT int _is_default_descr(PyObject *descr, PyObject *typestr) { - PyObject *tuple, *name, *typestr2; - PyObject *tmp = NULL; - int ret = 0; - if (!PyList_Check(descr) || PyList_GET_SIZE(descr) != 1) { return 0; } - tuple = PyList_GET_ITEM(descr, 0); + PyObject *tuple = PyList_GET_ITEM(descr, 0); if (!(PyTuple_Check(tuple) && PyTuple_GET_SIZE(tuple) == 2)) { return 0; } - name = PyTuple_GET_ITEM(tuple, 0); + PyObject *name = PyTuple_GET_ITEM(tuple, 0); if (!(PyUnicode_Check(name) && PyUnicode_GetLength(name) == 0)) { return 0; } - typestr2 = PyTuple_GET_ITEM(tuple, 1); - /* Allow unicode type strings */ - if (PyUnicode_Check(typestr2)) { - tmp = PyUnicode_AsASCIIString(typestr2); - if (tmp == NULL) { - return 0; - } - typestr2 = tmp; - } - if (PyBytes_Check(typestr2) && - PyObject_RichCompareBool(typestr, typestr2, Py_EQ)) { - ret = 1; - } - Py_XDECREF(tmp); - - return ret; + PyObject *typestr2 = PyTuple_GET_ITEM(tuple, 1); + return PyObject_RichCompareBool(typestr, typestr2, Py_EQ); } /*NUMPY_API*/ @@ -1893,26 +1875,15 @@ PyArray_FromInterface(PyObject *origin) return NULL; } - /* Allow unicode type strings */ - if (PyUnicode_Check(attr)) { - PyObject *tmp = PyUnicode_AsASCIIString(attr); - if (tmp == NULL) { - goto fail; - } - attr = tmp; - } - else { - Py_INCREF(attr); - } - - if (!PyBytes_Check(attr)) { + /* allow bytes for backwards compatibility */ + if (!PyBytes_Check(attr) && !PyUnicode_Check(attr)) { PyErr_SetString(PyExc_TypeError, "__array_interface__ typestr must be a string"); goto fail; } + /* Get dtype from type string */ - dtype = _array_typedescr_fromstr(PyString_AS_STRING(attr)); - if (dtype == NULL) { + if (PyArray_DescrConverter(attr, &dtype) != NPY_SUCCEED) { goto fail; } @@ -1926,16 +1897,24 @@ PyArray_FromInterface(PyObject *origin) goto fail; } PyArray_Descr *new_dtype = NULL; + if (descr != NULL) { + int is_default = _is_default_descr(descr, attr); + if (is_default < 0) { + goto fail; + } + if (!is_default) { + if (PyArray_DescrConverter2(descr, &new_dtype) != NPY_SUCCEED) { + goto fail; + } + if (new_dtype != NULL) { + Py_DECREF(dtype); + dtype = new_dtype; + } + } - if (descr != NULL && !_is_default_descr(descr, attr) && - PyArray_DescrConverter2(descr, &new_dtype) == NPY_SUCCEED && - new_dtype != NULL) { - Py_DECREF(dtype); - dtype = new_dtype; } - } - Py_DECREF(attr); /* Pairs with the unicode handling above */ + } /* Get shape tuple from interface specification */ attr = _PyDict_GetItemStringWithError(iface, "shape"); diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 03f10bf2d..a701de7c1 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -7436,6 +7436,18 @@ def test_array_interface_offset(): arr1 = np.asarray(DummyArray()) assert_equal(arr1, arr[1:]) +def test_array_interface_unicode_typestr(): + arr = np.array([1, 2, 3], dtype='int32') + interface = dict(arr.__array_interface__) + interface['typestr'] = '\N{check mark}' + + class DummyArray: + __array_interface__ = interface + + # should not be UnicodeEncodeError + with pytest.raises(TypeError): + np.asarray(DummyArray()) + def test_flat_element_deletion(): it = np.ones(3).flat try: -- cgit v1.2.1 From aa3570b296fb4d7d467c5185b7bdb96621b3d66b Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 25 Aug 2020 15:12:58 +0100 Subject: MAINT: remove _array_typedescr_fromstr and adjust its remaining caller --- numpy/core/src/multiarray/common.c | 20 -------------------- numpy/core/src/multiarray/ctors.c | 18 +++++++++++------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 5f8250fb7..2256906eb 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -127,26 +127,6 @@ PyArray_DTypeFromObject(PyObject *obj, int maxdims, PyArray_Descr **out_dtype) return 0; } - -/* new reference */ -NPY_NO_EXPORT PyArray_Descr * -_array_typedescr_fromstr(char const *c_str) -{ - PyArray_Descr *descr = NULL; - PyObject *stringobj = PyBytes_FromString(c_str); - - if (stringobj == NULL) { - return NULL; - } - if (PyArray_DescrConverter(stringobj, &descr) != NPY_SUCCEED) { - Py_DECREF(stringobj); - return NULL; - } - Py_DECREF(stringobj); - return descr; -} - - NPY_NO_EXPORT char * index2ptr(PyArrayObject *mp, npy_intp i) { diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index c9b1614f4..7534c0717 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1736,10 +1736,8 @@ NPY_NO_EXPORT PyObject * PyArray_FromStructInterface(PyObject *input) { PyArray_Descr *thetype = NULL; - char buf[40]; PyArrayInterface *inter; PyObject *attr; - PyArrayObject *ret; char endian = NPY_NATBYTE; attr = PyArray_LookupSpecial_OnInstance(input, "__array_struct__"); @@ -1782,20 +1780,26 @@ PyArray_FromStructInterface(PyObject *input) } if (thetype == NULL) { - PyOS_snprintf(buf, sizeof(buf), - "%c%c%d", endian, inter->typekind, inter->itemsize); - if (!(thetype=_array_typedescr_fromstr(buf))) { + PyObject *type_str = PyUnicode_FromFormat( + "%c%c%d", endian, inter->typekind, inter->itemsize); + if (type_str == NULL) { + Py_DECREF(attr); + return NULL; + } + int ok = PyArray_DescrConverter(type_str, &thetype); + Py_DECREF(type_str); + if (ok != NPY_SUCCEED) { Py_DECREF(attr); return NULL; } } - ret = (PyArrayObject *)PyArray_NewFromDescrAndBase( + PyObject *ret = PyArray_NewFromDescrAndBase( &PyArray_Type, thetype, inter->nd, inter->shape, inter->strides, inter->data, inter->flags, NULL, input); Py_DECREF(attr); - return (PyObject *)ret; + return ret; fail: PyErr_SetString(PyExc_ValueError, "invalid __array_struct__"); -- cgit v1.2.1 From be40e3f885b6a93f23edad20c5debe9379134b58 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Tue, 25 Aug 2020 15:18:48 +0100 Subject: MAINT: replace all remaining PyString_AS_STRING with PyBytes_AS_STRING These all look like valid enough uses of bytes strings --- numpy/core/src/multiarray/multiarraymodule.c | 6 +++--- numpy/core/src/multiarray/scalarapi.c | 2 +- numpy/core/src/umath/_rational_tests.c.src | 5 +++-- numpy/core/src/umath/_umath_tests.c.src | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index db419636d..c79d9a845 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1956,9 +1956,9 @@ array_scalar(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) return NULL; } } - if (!PyString_Check(obj)) { + if (!PyBytes_Check(obj)) { PyErr_SetString(PyExc_TypeError, - "initializing object must be a string"); + "initializing object must be a bytes object"); Py_XDECREF(tmpobj); return NULL; } @@ -1968,7 +1968,7 @@ array_scalar(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) Py_XDECREF(tmpobj); return NULL; } - dptr = PyString_AS_STRING(obj); + dptr = PyBytes_AS_STRING(obj); } } ret = PyArray_Scalar(dptr, typecode, NULL); diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index bcd777183..b2f52f554 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -138,7 +138,7 @@ scalar_value(PyObject *scalar, PyArray_Descr *descr) } else if (_CHK(Flexible)) { if (_CHK(String)) { - return (void *)PyString_AS_STRING(scalar); + return (void *)PyBytes_AS_STRING(scalar); } if (_CHK(Unicode)) { /* Treat this the same as the NPY_UNICODE base class */ diff --git a/numpy/core/src/umath/_rational_tests.c.src b/numpy/core/src/umath/_rational_tests.c.src index cbb6d9d17..e611a0847 100644 --- a/numpy/core/src/umath/_rational_tests.c.src +++ b/numpy/core/src/umath/_rational_tests.c.src @@ -406,8 +406,9 @@ pyrational_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { Py_INCREF(x[0]); return x[0]; } - else if (PyString_Check(x[0])) { - const char* s = PyString_AS_STRING(x[0]); + // TODO: allow construction from unicode strings + else if (PyBytes_Check(x[0])) { + const char* s = PyBytes_AS_STRING(x[0]); rational x; if (scan_rational(&s,&x)) { const char* p; diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index 932c3b5ab..3ab89d6a5 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -480,7 +480,7 @@ UMath_Tests_test_signature(PyObject *NPY_UNUSED(dummy), PyObject *args) return NULL; } - if (PyString_Check(signature)) { + if (PyBytes_Check(signature)) { sig_str = signature; } else if (PyUnicode_Check(signature)) { sig_str = PyUnicode_AsUTF8String(signature); @@ -493,7 +493,7 @@ UMath_Tests_test_signature(PyObject *NPY_UNUSED(dummy), PyObject *args) NULL, NULL, NULL, 0, nin, nout, PyUFunc_None, "no name", "doc:none", - 1, PyString_AS_STRING(sig_str)); + 1, PyBytes_AS_STRING(sig_str)); if (sig_str != signature) { Py_DECREF(sig_str); } -- cgit v1.2.1 From 051198414ba1e2c3e56919013d4d4372c34aa994 Mon Sep 17 00:00:00 2001 From: mattip Date: Thu, 27 Aug 2020 19:35:52 +0300 Subject: BUG: revert trim_zeros changes from gh-16911 --- doc/release/upcoming_changes/16911.deprecation.rst | 7 --- numpy/core/tests/test_deprecations.py | 21 -------- numpy/lib/function_base.py | 56 ---------------------- numpy/lib/tests/test_function_base.py | 4 ++ 4 files changed, 4 insertions(+), 84 deletions(-) delete mode 100644 doc/release/upcoming_changes/16911.deprecation.rst diff --git a/doc/release/upcoming_changes/16911.deprecation.rst b/doc/release/upcoming_changes/16911.deprecation.rst deleted file mode 100644 index 8f38ed989..000000000 --- a/doc/release/upcoming_changes/16911.deprecation.rst +++ /dev/null @@ -1,7 +0,0 @@ -``trim_zeros`` now requires a 1D array compatible with ``ndarray.astype(bool)`` -------------------------------------------------------------------------------- -The ``trim_zeros`` function will, in the future, require an array with the -following two properties: - -* It must be 1D. -* It must support elementwise comparisons with zero. diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index f0eac24ee..4ecfb0919 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -707,24 +707,3 @@ class TestRaggedArray(_DeprecationTestCase): self.assert_deprecated(lambda: np.array([arr, [0]], dtype=np.float64)) self.assert_deprecated(lambda: np.array([[0], arr], dtype=np.float64)) - -class TestTrimZeros(_DeprecationTestCase): - # Numpy 1.20.0, 2020-07-31 - @pytest.mark.parametrize( - "arr,exc_type", - [(np.random.rand(10, 10).tolist(), ValueError), - (np.random.rand(10).astype(str), FutureWarning)] - ) - def test_deprecated(self, arr, exc_type): - with warnings.catch_warnings(): - warnings.simplefilter('error', DeprecationWarning) - try: - np.trim_zeros(arr) - except DeprecationWarning as ex: - ex_cause = ex.__cause__ - assert_(isinstance(ex_cause, exc_type)) - else: - raise AssertionError("No error raised during function call") - - out = np.lib.function_base._trim_zeros_old(arr) - assert_array_equal(arr, out) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 0db00a0f2..710091de2 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -1625,63 +1625,7 @@ def trim_zeros(filt, trim='fb'): [1, 2] """ - try: - return _trim_zeros_new(filt, trim) - except Exception as ex: - # Numpy 1.20.0, 2020-07-31 - warning = DeprecationWarning( - "in the future trim_zeros will require a 1-D array as input " - "that supports elementwise comparisons with zero" - ) - warning.__cause__ = ex - - # Fall back to the old implementation if an exception is encountered - # Note that the same exception may or may not be raised here as well - ret = _trim_zeros_old(filt, trim) - warnings.warn(warning, stacklevel=3) - return ret - - -def _trim_zeros_new(filt, trim='fb'): - """Newer optimized implementation of ``trim_zeros()``.""" - arr_any = np.asanyarray(filt) - arr = arr_any != 0 if arr_any.dtype != bool else arr_any - - if arr is False: - # not all dtypes support elementwise comparisons with `0` (e.g. str); - # they will return `False` instead - raise TypeError('elementwise comparison failed; unsupported data type') - elif arr.ndim != 1: - raise ValueError('trim_zeros requires an array of exactly one dimension') - elif not len(arr): - return filt - - trim_upper = trim.upper() - first = last = None - - if 'F' in trim_upper: - first = arr.argmax() - # If `arr[first] is False` then so are all other elements - if not arr[first]: - return filt[:0] - if 'B' in trim_upper: - last = len(arr) - arr[::-1].argmax() - # If `arr[last - 1] is False` then so are all other elements - if not arr[last - 1]: - return filt[:0] - - return filt[first:last] - - -def _trim_zeros_old(filt, trim='fb'): - """ - Older unoptimized implementation of ``trim_zeros()``. - - Used as fallback in case an exception is encountered - in ``_trim_zeros_new()``. - - """ first = 0 trim = trim.upper() if 'F' in trim: diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 34a395ee4..41afccacc 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1225,6 +1225,10 @@ class TestTrimZeros: assert_array_equal(arr, res) + def test_list_to_list(self): + res = trim_zeros(self.a.tolist()) + assert isinstance(res, list) + class TestExtins: def test_basic(self): -- cgit v1.2.1 From e22696c9f675d5d977ebaa21f4bc7e8fbd67e8e6 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 28 Aug 2020 09:19:45 +0100 Subject: MAINT: Simplify error message formatting, push declarations down-file --- numpy/core/src/multiarray/datetime.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 348473309..903a28ffe 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1764,22 +1764,16 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, PyArray_DatetimeMetaData *out_meta, npy_bool from_pickle) { - char *basestr = NULL; - Py_ssize_t len = 0, tuple_size; int den = 1; - PyObject *unit_str = NULL; if (!PyTuple_Check(tuple)) { - PyObject *errmsg; - errmsg = PyUnicode_FromString("Require tuple for tuple to NumPy " - "datetime metadata conversion, not "); - PyUString_ConcatAndDel(&errmsg, PyObject_Repr(tuple)); - PyErr_SetObject(PyExc_TypeError, errmsg); - Py_DECREF(errmsg); + PyErr_Format(PyExc_TypeError, + "Require tuple for tuple to NumPy " + "datetime metadata conversion, not %R", tuple); return -1; } - tuple_size = PyTuple_GET_SIZE(tuple); + Py_ssize_t tuple_size = PyTuple_GET_SIZE(tuple); if (tuple_size < 2 || tuple_size > 4) { PyErr_SetString(PyExc_TypeError, "Require tuple of size 2 to 4 for " @@ -1787,7 +1781,7 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, return -1; } - unit_str = PyTuple_GET_ITEM(tuple, 0); + PyObject *unit_str = PyTuple_GET_ITEM(tuple, 0); Py_INCREF(unit_str); if (PyUnicode_Check(unit_str)) { /* Allow unicode format strings: convert to bytes */ @@ -1798,6 +1792,9 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, } unit_str = tmp; } + + Py_ssize_t len; + char *basestr; if (PyBytes_AsStringAndSize(unit_str, &basestr, &len) < 0) { Py_DECREF(unit_str); return -1; @@ -1837,11 +1834,10 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, if (from_pickle) { /* if (event == 1) */ PyObject *one = PyLong_FromLong(1); - int equal_one; if (one == NULL) { return -1; } - equal_one = PyObject_RichCompareBool(event, one, Py_EQ); + int equal_one = PyObject_RichCompareBool(event, one, Py_EQ); Py_DECREF(one); if (equal_one == -1) { return -1; -- cgit v1.2.1 From 80701dda0fd7b16ecccc2243e9a0c5f00194fef6 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 28 Aug 2020 09:32:09 +0100 Subject: MAINT: Produce a better error message for unicode datetime formats Rather than raising UnicodeEncodeError, this just raises ValueError. Passing weird byte strings results in UnicodeDecodeError instead. In future, we may just want to deprecate passing `bytes` objects. --- numpy/core/src/multiarray/datetime.c | 106 ++++++++++++++++++----------------- numpy/core/tests/test_datetime.py | 8 +++ 2 files changed, 62 insertions(+), 52 deletions(-) diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 903a28ffe..941af0bf9 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1782,20 +1782,21 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, } PyObject *unit_str = PyTuple_GET_ITEM(tuple, 0); - Py_INCREF(unit_str); - if (PyUnicode_Check(unit_str)) { - /* Allow unicode format strings: convert to bytes */ - PyObject *tmp = PyUnicode_AsASCIIString(unit_str); - Py_DECREF(unit_str); + if (PyBytes_Check(unit_str)) { + /* Allow bytes format strings: convert to unicode */ + PyObject *tmp = PyUnicode_FromEncodedObject(unit_str, NULL, NULL); if (tmp == NULL) { return -1; } unit_str = tmp; } + else { + Py_INCREF(unit_str); + } Py_ssize_t len; - char *basestr; - if (PyBytes_AsStringAndSize(unit_str, &basestr, &len) < 0) { + char const *basestr = PyUnicode_AsUTF8AndSize(unit_str, &len); + if (basestr == NULL) { Py_DECREF(unit_str); return -1; } @@ -1896,26 +1897,23 @@ NPY_NO_EXPORT int convert_pyobject_to_datetime_metadata(PyObject *obj, PyArray_DatetimeMetaData *out_meta) { - PyObject *ascii = NULL; - char *str = NULL; - Py_ssize_t len = 0; - if (PyTuple_Check(obj)) { return convert_datetime_metadata_tuple_to_datetime_metadata( obj, out_meta, NPY_FALSE); } - /* Get an ASCII string */ - if (PyUnicode_Check(obj)) { - /* Allow unicode format strings: convert to bytes */ - ascii = PyUnicode_AsASCIIString(obj); - if (ascii == NULL) { + /* Get a UTF8 string */ + PyObject *utf8 = NULL; + if (PyBytes_Check(obj)) { + /* Allow bytes format strings: convert to unicode */ + utf8 = PyUnicode_FromEncodedObject(obj, NULL, NULL); + if (utf8 == NULL) { return -1; } } - else if (PyBytes_Check(obj)) { - ascii = obj; - Py_INCREF(ascii); + else if (PyUnicode_Check(obj)) { + utf8 = obj; + Py_INCREF(utf8); } else { PyErr_SetString(PyExc_TypeError, @@ -1923,24 +1921,26 @@ convert_pyobject_to_datetime_metadata(PyObject *obj, return -1; } - if (PyBytes_AsStringAndSize(ascii, &str, &len) < 0) { - Py_DECREF(ascii); + Py_ssize_t len = 0; + char const *str = PyUnicode_AsUTF8AndSize(utf8, &len); + if (str == NULL) { + Py_DECREF(utf8); return -1; } if (len > 0 && str[0] == '[') { int r = parse_datetime_metadata_from_metastr(str, len, out_meta); - Py_DECREF(ascii); + Py_DECREF(utf8); return r; } else { if (parse_datetime_extended_unit_from_string(str, len, NULL, out_meta) < 0) { - Py_DECREF(ascii); + Py_DECREF(utf8); return -1; } - Py_DECREF(ascii); + Py_DECREF(utf8); return 0; } } @@ -2346,32 +2346,33 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, NPY_CASTING casting, npy_datetime *out) { if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { - PyObject *bytes = NULL; - char *str = NULL; - Py_ssize_t len = 0; - npy_datetimestruct dts; - NPY_DATETIMEUNIT bestunit = NPY_FR_ERROR; + PyObject *utf8 = NULL; - /* Convert to an ASCII string for the date parser */ - if (PyUnicode_Check(obj)) { - bytes = PyUnicode_AsASCIIString(obj); - if (bytes == NULL) { + /* Convert to an UTF8 string for the date parser */ + if (PyBytes_Check(obj)) { + utf8 = PyUnicode_FromEncodedObject(obj, NULL, NULL); + if (utf8 == NULL) { return -1; } } else { - bytes = obj; - Py_INCREF(bytes); + utf8 = obj; + Py_INCREF(utf8); } - if (PyBytes_AsStringAndSize(bytes, &str, &len) < 0) { - Py_DECREF(bytes); + + Py_ssize_t len = 0; + char const *str = PyUnicode_AsUTF8AndSize(utf8, &len); + if (str == NULL) { + Py_DECREF(utf8); return -1; } /* Parse the ISO date */ + npy_datetimestruct dts; + NPY_DATETIMEUNIT bestunit = NPY_FR_ERROR; if (parse_iso_8601_datetime(str, len, meta->base, casting, &dts, &bestunit, NULL) < 0) { - Py_DECREF(bytes); + Py_DECREF(utf8); return -1; } @@ -2382,11 +2383,11 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } if (convert_datetimestruct_to_datetime(meta, &dts, out) < 0) { - Py_DECREF(bytes); + Py_DECREF(utf8); return -1; } - Py_DECREF(bytes); + Py_DECREF(utf8); return 0; } /* Do no conversion on raw integers */ @@ -2540,24 +2541,25 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, NPY_CASTING casting, npy_timedelta *out) { if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { - PyObject *bytes = NULL; - char *str = NULL; - Py_ssize_t len = 0; + PyObject *utf8 = NULL; int succeeded = 0; - /* Convert to an ASCII string for the date parser */ - if (PyUnicode_Check(obj)) { - bytes = PyUnicode_AsASCIIString(obj); - if (bytes == NULL) { + /* Convert to an UTF8 string for the date parser */ + if (PyBytes_Check(obj)) { + utf8 = PyUnicode_FromEncodedObject(obj, NULL, NULL); + if (utf8 == NULL) { return -1; } } else { - bytes = obj; - Py_INCREF(bytes); + utf8 = obj; + Py_INCREF(utf8); } - if (PyBytes_AsStringAndSize(bytes, &str, &len) < 0) { - Py_DECREF(bytes); + + Py_ssize_t len = 0; + char const *str = PyUnicode_AsUTF8AndSize(utf8, &len); + if (str == NULL) { + Py_DECREF(utf8); return -1; } @@ -2578,7 +2580,7 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, succeeded = 1; } } - Py_DECREF(bytes); + Py_DECREF(utf8); if (succeeded) { /* Use generic units if none was specified */ diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 59a3954fd..a546daa58 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -2389,3 +2389,11 @@ class TestDateTimeData: def test_basic(self): a = np.array(['1980-03-23'], dtype=np.datetime64) assert_equal(np.datetime_data(a.dtype), ('D', 1)) + + def test_bytes(self): + # byte units are converted to unicode + dt = np.datetime64('2000', (b'ms', 5)) + assert np.datetime_data(dt.dtype) == ('ms', 5) + + dt = np.datetime64('2000', b'5ms') + assert np.datetime_data(dt.dtype) == ('ms', 5) -- cgit v1.2.1 From d3e4792a6ef9532825d481586dc38c0e2e1eb53b Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Fri, 28 Aug 2020 09:46:51 +0100 Subject: =?UTF-8?q?ENH:=20Allow=20=CE=BCs=20as=20an=20alias=20for=20us?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is motivated primarily as a proof that the datetime API is now unicode-safe, and generally seems harmless. --- doc/source/reference/arrays.datetime.rst | 2 +- numpy/core/src/multiarray/datetime.c | 4 ++++ numpy/core/tests/test_datetime.py | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/source/reference/arrays.datetime.rst b/doc/source/reference/arrays.datetime.rst index 9ce77424a..c5947620e 100644 --- a/doc/source/reference/arrays.datetime.rst +++ b/doc/source/reference/arrays.datetime.rst @@ -218,7 +218,7 @@ And here are the time units: m minute +/- 1.7e13 years [1.7e13 BC, 1.7e13 AD] s second +/- 2.9e11 years [2.9e11 BC, 2.9e11 AD] ms millisecond +/- 2.9e8 years [ 2.9e8 BC, 2.9e8 AD] - us microsecond +/- 2.9e5 years [290301 BC, 294241 AD] +us / μs microsecond +/- 2.9e5 years [290301 BC, 294241 AD] ns nanosecond +/- 292 years [ 1678 AD, 2262 AD] ps picosecond +/- 106 days [ 1969 AD, 1970 AD] fs femtosecond +/- 2.6 hours [ 1969 AD, 1970 AD] diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 941af0bf9..f2225809a 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1717,6 +1717,10 @@ parse_datetime_unit_from_string(char const *str, Py_ssize_t len, char const *met return NPY_FR_as; } } + else if (len == 3 && !strncmp(str, "\xce\xbcs", 3)) { + /* greek small letter mu, utf8-encoded */ + return NPY_FR_us; + } else if (len == 7 && !strncmp(str, "generic", 7)) { return NPY_FR_GENERIC; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index a546daa58..f725091c5 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -26,6 +26,7 @@ class TestDateTime: def test_datetime_dtype_creation(self): for unit in ['Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', + 'μs', # alias for us 'ns', 'ps', 'fs', 'as']: dt1 = np.dtype('M8[750%s]' % unit) assert_(dt1 == np.dtype('datetime64[750%s]' % unit)) @@ -2397,3 +2398,11 @@ class TestDateTimeData: dt = np.datetime64('2000', b'5ms') assert np.datetime_data(dt.dtype) == ('ms', 5) + + def test_non_ascii(self): + # μs is normalized to μ + dt = np.datetime64('2000', ('μs', 5)) + assert np.datetime_data(dt.dtype) == ('us', 5) + + dt = np.datetime64('2000', '5μs') + assert np.datetime_data(dt.dtype) == ('us', 5) -- cgit v1.2.1 From 4cd6e4b336fbc68d88c0e9bc45a435ce7b721f1f Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Fri, 28 Aug 2020 20:05:18 +0200 Subject: ENH: implement NEP-35's `like=` argument (gh-16935) This PR adds the implementation of NEP-35's like= argument, allowing dispatch of array creation functions with __array_function__ based on a reference array. * ENH: Add like= kwarg via __array_function__ dispatcher to asarray * ENH: Add new function for __array_function__ dispatching from C This new function allows dispatching from C directly, while also implementing the new `like=` argument, requiring only minimal changes to existing array creation functions that need to add support for that argument. * ENH: Add like= support to numpy.array The implementation uses array_implement_c_array_function, thus introducing minimal complexity to the original _array_fromobject code. * BUG: Fix like= dispatcher for np.full * ENH: Remove np.asarray like= dispatcher via Python np.asarray can rely on np.array's C dispatcher instead. * TST: Add some tests for like= argument Tests comprise some of the functions that have been implemented already: * np.array (C dispatcher) * np.asarray (indirect C dispatcher via np.array) * np.full (Python dispatcher) * np.ones (Python dispatcher) * ENH: Remove like= argument during array_implement_array_function * ENH: Add like= kwarg to ones and full * BUG: prevent duplicate removal of `like=` argument * ENH: Make `like=` a keyword-only argument * ENH: Use PyUnicode_InternFromString in arrayfunction_override Replace PyUnicode_FromString by PyUnicode_InternFromString to cache "like" string. * ENH: Check for arrayfunction_override import errors Check and handle errors on importing NumPy's Python functions * BUG: Fix array_implement_c_array_function error handling * ENH: Handle exceptions with C implementation of `like=` * ENH: Add `like=` dispatch for all asarray functions Using Python dispatcher for all of them. Using the C dispatcher directly on the `np.array` call can result in incorrect behavior. Incorrect behavior may happen if the downstream library's implementation is different or if not all keyword arguments are supported. * ENH: Simplify handling of exceptions with `like=` * TST: Add test for exception handling with `like=` * ENH: Add support for `like=` to `np.empty` and `np.zeros` * TST: Add `like=` tests for `np.empty` and `np.zeros` * ENH: Add `like=` to remaining multiarraymodule.c functions Functions are: * np.arange * np.frombuffer * np.fromfile * np.fromiter * np.fromstring * TST: Add tests for multiarraymodule.c functions with like= Functions are: * np.arange * np.frombuffer * np.fromfile * np.fromiter * np.fromstring * ENH: Add `like=` support to more creation functions Support for the following functions is added: * np.eye * np.fromfunction * np.genfromtxt * np.identity * np.loadtxt * np.tri * TST: Add `like=` tests for multiple functions Tests for the following functions are added: * np.eye * np.fromfunction * np.genfromtxt * np.identity * np.loadtxt * np.tri * TST: Reduce code duplication in `like=` tests * DOC: Document `like=` in functions that support it Add documentations for the following functions: * np.array * np.arange * np.asarray * np.asanyarray * np.ascontiguousarray * np.asfortranarray * np.require * np.empty * np.full * np.ones * np.zeros * np.identity * np.eye * np.tri * np.frombuffer * np.fromfile * np.fromiter * np.fromstring * np.loadtxt * np.genfromtxt * ENH: Add `like=` to numpy/__init__.pyi stubs * BUG: Remove duplicate `like=` dispatching in as*array Functions `np.asanyarray`, `np.contiguousarray` and `np.fortranarray` were dispatching both via their definitions and `np.array` calls, the latter should be avoided. * BUG: Fix missing check in array_implement_array_function * BUG: Add missing keyword-only markers in stubs * BUG: Fix duplicate keyword-only marker in array stub * BUG: Fix syntax error in numpy/__init__.pyi * BUG: Fix more syntax errors in numpy/__init__.pyi * ENH: Intern arrayfunction_override strings in multiarraymodule * STY: Add missing brackets to arrayfunction_override.c * MAINT: Remove arrayfunction_override dict check for kwarg * TST: Assert that 'like' is not in TestArrayLike kwargs * MAINT: Rename array_implement_c_array_function(_creation) This is done to be more explicit as to its usage being intended for array creation functions only. * MAINT: Use NotImplemented to indicate fallback to default * TST: Test that results with `like=np.array` are correct * TST: Avoid duplicating MyArray code in TestArrayLike * TST: Don't delete temp file, it may cause issues with Windows * TST: Don't rely on eval in TestArrayLike * TST: Use lambda with StringIO in TestArrayLike * ENH: Avoid unnecessary Py_XDECREF in arrayfunction_override * TST: Make TestArrayLike more readable * ENH: Cleaner error handling in arrayfunction_override * ENH: Simplify array_implement_c_array_function_creation * STY: Add missing spaces to multiarraymodule.c * STY: C99 declaration style in arrayfunction_override.c * ENH: Simplify arrayfunction_override.c further Remove cleanup label from array_implementation_c_array_function, simplifying the code. Fix unitialized variable warning in array_implementation_array_function_internal. * DOC: Use string replacement for `like=` documentation Avoid repeating the full text for the `like=` argument by storing it as a variable and using `replace` on each docstring. * DOC: Update `like=` docstring * TST: Test like= with array not implementing __array_function__ * TST: Add missing asanyarray to TestArrayLike * ENH: Use helper function for like= dispatching Avoid dispatching like= from Python implementation functions to improve their performance. This is achieved by only calling a dispatcher function when like is passed by the users. * ENH: Rename array_function_dispatch kwarg to public_api * BUG: Add accidentally removed decorator for np.eye back * DOC: Add set_array_function_like_doc function The function keeps Python files cleaner and resolve errors when __doc__ is not defined due to PYTHONOPTIMIZE or -OO . * DOC: Add mention to like= kwarg being experimental * TST: Test like= with not implemented downstream function * DOC: Fix like= docstring reference to NEP 35. * ENH: Prevent silent errors if public_api is not callable * ENH: Make set_array_function_like_doc a decorator * ENH: Simplify `_*_with_like` functions * BUG: Fix multiple `like=` dispatching in `require` * MAINT: Remove now unused public_api from array_function_dispatch Co-authored-by: Sebastian Berg --- numpy/__init__.pyi | 36 ++++- numpy/core/_add_newdocs.py | 82 ++++++++-- numpy/core/_asarray.py | 94 +++++++++++- numpy/core/numeric.py | 76 ++++++++- numpy/core/overrides.py | 21 +++ numpy/core/src/multiarray/arrayfunction_override.c | 170 +++++++++++++++------ numpy/core/src/multiarray/arrayfunction_override.h | 4 + numpy/core/src/multiarray/multiarraymodule.c | 129 ++++++++++++---- numpy/core/src/multiarray/multiarraymodule.h | 2 + numpy/core/tests/test_overrides.py | 167 ++++++++++++++++++++ numpy/lib/npyio.py | 65 +++++++- numpy/lib/twodim_base.py | 37 ++++- 12 files changed, 772 insertions(+), 111 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index c6cc94440..17f7dda67 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -524,15 +524,28 @@ def array( order: Optional[str] = ..., subok: bool = ..., ndmin: int = ..., + like: ArrayLike = ..., ) -> ndarray: ... def zeros( - shape: _ShapeLike, dtype: DtypeLike = ..., order: Optional[str] = ... + shape: _ShapeLike, + dtype: DtypeLike = ..., + order: Optional[str] = ..., + *, + like: ArrayLike = ..., ) -> ndarray: ... def ones( - shape: _ShapeLike, dtype: DtypeLike = ..., order: Optional[str] = ... + shape: _ShapeLike, + dtype: DtypeLike = ..., + order: Optional[str] = ..., + *, + like: ArrayLike = ..., ) -> ndarray: ... def empty( - shape: _ShapeLike, dtype: DtypeLike = ..., order: Optional[str] = ... + shape: _ShapeLike, + dtype: DtypeLike = ..., + order: Optional[str] = ..., + *, + like: ArrayLike = ..., ) -> ndarray: ... def zeros_like( a: ArrayLike, @@ -556,7 +569,12 @@ def empty_like( shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... def full( - shape: _ShapeLike, fill_value: Any, dtype: DtypeLike = ..., order: str = ... + shape: _ShapeLike, + fill_value: Any, + dtype: DtypeLike = ..., + order: str = ..., + *, + like: ArrayLike = ..., ) -> ndarray: ... def full_like( a: ArrayLike, @@ -604,11 +622,17 @@ def cross( def indices( dimensions: Sequence[int], dtype: dtype = ..., sparse: bool = ... ) -> Union[ndarray, Tuple[ndarray, ...]]: ... -def fromfunction(function: Callable, shape: Tuple[int, int], **kwargs) -> Any: ... +def fromfunction( + function: Callable, + shape: Tuple[int, int], + *, + like: ArrayLike = ..., + **kwargs, +) -> Any: ... def isscalar(element: Any) -> bool: ... def binary_repr(num: int, width: Optional[int] = ...) -> str: ... def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... -def identity(n: int, dtype: DtypeLike = ...) -> ndarray: ... +def identity(n: int, dtype: DtypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... def allclose( a: ArrayLike, b: ArrayLike, diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index c3b4374f4..f11f008b2 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -12,6 +12,7 @@ NOTE: Many of the methods of ndarray have corresponding functions. from numpy.core import numerictypes as _numerictypes from numpy.core import dtype from numpy.core.function_base import add_newdoc +from numpy.core.overrides import array_function_like_doc ############################################################################### # @@ -786,7 +787,8 @@ add_newdoc('numpy.core', 'broadcast', ('reset', add_newdoc('numpy.core.multiarray', 'array', """ - array(object, dtype=None, *, copy=True, order='K', subok=False, ndmin=0) + array(object, dtype=None, *, copy=True, order='K', subok=False, ndmin=0, + like=None) Create an array. @@ -829,6 +831,9 @@ add_newdoc('numpy.core.multiarray', 'array', Specifies the minimum number of dimensions that the resulting array should have. Ones will be pre-pended to the shape as needed to meet this requirement. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -895,11 +900,14 @@ add_newdoc('numpy.core.multiarray', 'array', matrix([[1, 2], [3, 4]]) - """) + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) add_newdoc('numpy.core.multiarray', 'empty', """ - empty(shape, dtype=float, order='C') + empty(shape, dtype=float, order='C', *, like=None) Return a new array of given shape and type, without initializing entries. @@ -914,6 +922,9 @@ add_newdoc('numpy.core.multiarray', 'empty', Whether to store multi-dimensional data in row-major (C-style) or column-major (Fortran-style) order in memory. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -946,7 +957,10 @@ add_newdoc('numpy.core.multiarray', 'empty', array([[-1073741821, -1067949133], [ 496041986, 19249760]]) #uninitialized - """) + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) add_newdoc('numpy.core.multiarray', 'scalar', """ @@ -964,7 +978,7 @@ add_newdoc('numpy.core.multiarray', 'scalar', add_newdoc('numpy.core.multiarray', 'zeros', """ - zeros(shape, dtype=float, order='C') + zeros(shape, dtype=float, order='C', *, like=None) Return a new array of given shape and type, filled with zeros. @@ -979,6 +993,9 @@ add_newdoc('numpy.core.multiarray', 'zeros', Whether to store multi-dimensional data in row-major (C-style) or column-major (Fortran-style) order in memory. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -1013,7 +1030,10 @@ add_newdoc('numpy.core.multiarray', 'zeros', array([(0, 0), (0, 0)], dtype=[('x', '>> np.fromstring('1, 2', dtype=int, sep=',') array([1, 2]) - """) + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) add_newdoc('numpy.core.multiarray', 'compare_chararrays', """ @@ -1122,7 +1148,7 @@ add_newdoc('numpy.core.multiarray', 'compare_chararrays', add_newdoc('numpy.core.multiarray', 'fromiter', """ - fromiter(iterable, dtype, count=-1) + fromiter(iterable, dtype, count=-1, *, like=None) Create a new 1-dimensional array from an iterable object. @@ -1135,6 +1161,9 @@ add_newdoc('numpy.core.multiarray', 'fromiter', count : int, optional The number of items to read from *iterable*. The default is -1, which means all data is read. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -1152,11 +1181,14 @@ add_newdoc('numpy.core.multiarray', 'fromiter', >>> np.fromiter(iterable, float) array([ 0., 1., 4., 9., 16.]) - """) + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) add_newdoc('numpy.core.multiarray', 'fromfile', """ - fromfile(file, dtype=float, count=-1, sep='', offset=0) + fromfile(file, dtype=float, count=-1, sep='', offset=0, *, like=None) Construct an array from data in a text or binary file. @@ -1195,6 +1227,9 @@ add_newdoc('numpy.core.multiarray', 'fromfile', Only permitted for binary files. .. versionadded:: 1.17.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 See also -------- @@ -1241,11 +1276,14 @@ add_newdoc('numpy.core.multiarray', 'fromfile', array([((10, 0), 98.25)], dtype=[('time', [('min', '>> np.frombuffer(b'\\x01\\x02\\x03\\x04\\x05', dtype=np.uint8, count=3) array([1, 2, 3], dtype=uint8) - """) + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) add_newdoc('numpy.core', 'fastCopyAndTranspose', """_fastCopyAndTranspose(a)""") @@ -1293,7 +1337,7 @@ add_newdoc('numpy.core.multiarray', 'correlate', add_newdoc('numpy.core.multiarray', 'arange', """ - arange([start,] stop[, step,], dtype=None) + arange([start,] stop[, step,], dtype=None, *, like=None) Return evenly spaced values within a given interval. @@ -1322,6 +1366,9 @@ add_newdoc('numpy.core.multiarray', 'arange', dtype : dtype The type of the output array. If `dtype` is not given, infer the data type from the other input arguments. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -1350,7 +1397,10 @@ add_newdoc('numpy.core.multiarray', 'arange', >>> np.arange(3,7,2) array([3, 5]) - """) + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) add_newdoc('numpy.core.multiarray', '_get_ndarray_c_version', """_get_ndarray_c_version() diff --git a/numpy/core/_asarray.py b/numpy/core/_asarray.py index 1b06c328f..a406308f3 100644 --- a/numpy/core/_asarray.py +++ b/numpy/core/_asarray.py @@ -3,7 +3,11 @@ Functions in the ``as*array`` family that promote array-likes into arrays. `require` fits this category despite its name not matching this pattern. """ -from .overrides import set_module +from .overrides import ( + array_function_dispatch, + set_array_function_like_doc, + set_module, +) from .multiarray import array @@ -11,8 +15,14 @@ __all__ = [ "asarray", "asanyarray", "ascontiguousarray", "asfortranarray", "require", ] + +def _asarray_dispatcher(a, dtype=None, order=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def asarray(a, dtype=None, order=None): +def asarray(a, dtype=None, order=None, *, like=None): """Convert the input to an array. Parameters @@ -30,6 +40,9 @@ def asarray(a, dtype=None, order=None): 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise 'K' (keep) preserve input order Defaults to 'C'. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -83,11 +96,20 @@ def asarray(a, dtype=None, order=None): True """ + if like is not None: + return _asarray_with_like(a, dtype=dtype, order=order, like=like) + return array(a, dtype, copy=False, order=order) +_asarray_with_like = array_function_dispatch( + _asarray_dispatcher +)(asarray) + + +@set_array_function_like_doc @set_module('numpy') -def asanyarray(a, dtype=None, order=None): +def asanyarray(a, dtype=None, order=None, *, like=None): """Convert the input to an ndarray, but pass ndarray subclasses through. Parameters @@ -105,6 +127,9 @@ def asanyarray(a, dtype=None, order=None): 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise 'K' (keep) preserve input order Defaults to 'C'. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -140,11 +165,24 @@ def asanyarray(a, dtype=None, order=None): True """ + if like is not None: + return _asanyarray_with_like(a, dtype=dtype, order=order, like=like) + return array(a, dtype, copy=False, order=order, subok=True) +_asanyarray_with_like = array_function_dispatch( + _asarray_dispatcher +)(asanyarray) + + +def _asarray_contiguous_fortran_dispatcher(a, dtype=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def ascontiguousarray(a, dtype=None): +def ascontiguousarray(a, dtype=None, *, like=None): """ Return a contiguous array (ndim >= 1) in memory (C order). @@ -154,6 +192,9 @@ def ascontiguousarray(a, dtype=None): Input array. dtype : str or dtype object, optional Data-type of returned array. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -181,11 +222,20 @@ def ascontiguousarray(a, dtype=None): so it will not preserve 0-d arrays. """ + if like is not None: + return _ascontiguousarray_with_like(a, dtype=dtype, like=like) + return array(a, dtype, copy=False, order='C', ndmin=1) +_ascontiguousarray_with_like = array_function_dispatch( + _asarray_contiguous_fortran_dispatcher +)(ascontiguousarray) + + +@set_array_function_like_doc @set_module('numpy') -def asfortranarray(a, dtype=None): +def asfortranarray(a, dtype=None, *, like=None): """ Return an array (ndim >= 1) laid out in Fortran order in memory. @@ -195,6 +245,9 @@ def asfortranarray(a, dtype=None): Input array. dtype : str or dtype object, optional By default, the data-type is inferred from the input data. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -222,11 +275,24 @@ def asfortranarray(a, dtype=None): so it will not preserve 0-d arrays. """ + if like is not None: + return _asfortranarray_with_like(a, dtype=dtype, like=like) + return array(a, dtype, copy=False, order='F', ndmin=1) +_asfortranarray_with_like = array_function_dispatch( + _asarray_contiguous_fortran_dispatcher +)(asfortranarray) + + +def _require_dispatcher(a, dtype=None, requirements=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def require(a, dtype=None, requirements=None): +def require(a, dtype=None, requirements=None, *, like=None): """ Return an ndarray of the provided type that satisfies requirements. @@ -250,6 +316,9 @@ def require(a, dtype=None, requirements=None): * 'WRITEABLE' ('W') - ensure a writable array * 'OWNDATA' ('O') - ensure an array that owns its own data * 'ENSUREARRAY', ('E') - ensure a base array, instead of a subclass + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -293,6 +362,14 @@ def require(a, dtype=None, requirements=None): UPDATEIFCOPY : False """ + if like is not None: + return _require_with_like( + a, + dtype=dtype, + requirements=requirements, + like=like, + ) + possible_flags = {'C': 'C', 'C_CONTIGUOUS': 'C', 'CONTIGUOUS': 'C', 'F': 'F', 'F_CONTIGUOUS': 'F', 'FORTRAN': 'F', 'A': 'A', 'ALIGNED': 'A', @@ -327,3 +404,8 @@ def require(a, dtype=None, requirements=None): arr = arr.copy(order) break return arr + + +_require_with_like = array_function_dispatch( + _require_dispatcher +)(require) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 84066dd30..a023bf0da 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -21,7 +21,7 @@ from .multiarray import ( from . import overrides from . import umath from . import shape_base -from .overrides import set_module +from .overrides import set_array_function_like_doc, set_module from .umath import (multiply, invert, sin, PINF, NAN) from . import numerictypes from .numerictypes import longlong, intc, int_, float_, complex_, bool_ @@ -141,8 +141,13 @@ def zeros_like(a, dtype=None, order='K', subok=True, shape=None): return res +def _ones_dispatcher(shape, dtype=None, order=None, *, like=None): + return(like,) + + +@set_array_function_like_doc @set_module('numpy') -def ones(shape, dtype=None, order='C'): +def ones(shape, dtype=None, order='C', *, like=None): """ Return a new array of given shape and type, filled with ones. @@ -157,6 +162,9 @@ def ones(shape, dtype=None, order='C'): Whether to store multi-dimensional data in row-major (C-style) or column-major (Fortran-style) order in memory. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -189,11 +197,19 @@ def ones(shape, dtype=None, order='C'): [1., 1.]]) """ + if like is not None: + return _ones_with_like(shape, dtype=dtype, order=order, like=like) + a = empty(shape, dtype, order) multiarray.copyto(a, 1, casting='unsafe') return a +_ones_with_like = array_function_dispatch( + _ones_dispatcher +)(ones) + + def _ones_like_dispatcher(a, dtype=None, order=None, subok=None, shape=None): return (a,) @@ -265,8 +281,13 @@ def ones_like(a, dtype=None, order='K', subok=True, shape=None): return res +def _full_dispatcher(shape, fill_value, dtype=None, order=None, *, like=None): + return(like,) + + +@set_array_function_like_doc @set_module('numpy') -def full(shape, fill_value, dtype=None, order='C'): +def full(shape, fill_value, dtype=None, order='C', *, like=None): """ Return a new array of given shape and type, filled with `fill_value`. @@ -282,6 +303,9 @@ def full(shape, fill_value, dtype=None, order='C'): order : {'C', 'F'}, optional Whether to store multidimensional data in C- or Fortran-contiguous (row- or column-wise) order in memory. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -309,6 +333,9 @@ def full(shape, fill_value, dtype=None, order='C'): [1, 2]]) """ + if like is not None: + return _full_with_like(shape, fill_value, dtype=dtype, order=order, like=like) + if dtype is None: fill_value = asarray(fill_value) dtype = fill_value.dtype @@ -317,6 +344,11 @@ def full(shape, fill_value, dtype=None, order='C'): return a +_full_with_like = array_function_dispatch( + _full_dispatcher +)(full) + + def _full_like_dispatcher(a, fill_value, dtype=None, order=None, subok=None, shape=None): return (a,) @@ -1754,8 +1786,13 @@ def indices(dimensions, dtype=int, sparse=False): return res +def _fromfunction_dispatcher(function, shape, *, dtype=None, like=None, **kwargs): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def fromfunction(function, shape, *, dtype=float, **kwargs): +def fromfunction(function, shape, *, dtype=float, like=None, **kwargs): """ Construct an array by executing a function over each coordinate. @@ -1776,6 +1813,9 @@ def fromfunction(function, shape, *, dtype=float, **kwargs): dtype : data-type, optional Data-type of the coordinate arrays passed to `function`. By default, `dtype` is float. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -1806,10 +1846,18 @@ def fromfunction(function, shape, *, dtype=float, **kwargs): [2, 3, 4]]) """ + if like is not None: + return _fromfunction_with_like(function, shape, dtype=dtype, like=like, **kwargs) + args = indices(shape, dtype=dtype) return function(*args, **kwargs) +_fromfunction_with_like = array_function_dispatch( + _fromfunction_dispatcher +)(fromfunction) + + def _frombuffer(buf, dtype, shape, order): return frombuffer(buf, dtype=dtype).reshape(shape, order=order) @@ -2082,8 +2130,13 @@ def _maketup(descr, val): return tuple(res) +def _identity_dispatcher(n, dtype=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def identity(n, dtype=None): +def identity(n, dtype=None, *, like=None): """ Return the identity array. @@ -2096,6 +2149,9 @@ def identity(n, dtype=None): Number of rows (and columns) in `n` x `n` output. dtype : data-type, optional Data-type of the output. Defaults to ``float``. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -2111,8 +2167,16 @@ def identity(n, dtype=None): [0., 0., 1.]]) """ + if like is not None: + return _identity_with_like(n, dtype=dtype, like=like) + from numpy import eye - return eye(n, dtype=dtype) + return eye(n, dtype=dtype, like=like) + + +_identity_with_like = array_function_dispatch( + _identity_dispatcher +)(identity) def _allclose_dispatcher(a, b, rtol=None, atol=None, equal_nan=None): diff --git a/numpy/core/overrides.py b/numpy/core/overrides.py index 816b11293..c2b5fb7fa 100644 --- a/numpy/core/overrides.py +++ b/numpy/core/overrides.py @@ -12,6 +12,27 @@ from numpy.compat._inspect import getargspec ARRAY_FUNCTION_ENABLED = bool( int(os.environ.get('NUMPY_EXPERIMENTAL_ARRAY_FUNCTION', 1))) +array_function_like_doc = ( + """like : array_like + Reference object to allow the creation of arrays which are not + NumPy arrays. If an array-like passed in as ``like`` supports + the ``__array_function__`` protocol, the result will be defined + by it. In this case, it ensures the creation of an array object + compatible with that passed in via this argument. + + .. note:: + The ``like`` keyword is an experimental feature pending on + acceptance of :ref:`NEP 35 `.""" +) + +def set_array_function_like_doc(public_api): + if public_api.__doc__ is not None: + public_api.__doc__ = public_api.__doc__.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + ) + return public_api + add_docstring( implement_array_function, diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c index 9ea8efdd9..613fe6b3f 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.c +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -26,7 +26,6 @@ static PyObject * get_array_function(PyObject *obj) { static PyObject *ndarray_array_function = NULL; - PyObject *array_function; if (ndarray_array_function == NULL) { ndarray_array_function = get_ndarray_array_function(); @@ -38,7 +37,7 @@ get_array_function(PyObject *obj) return ndarray_array_function; } - array_function = PyArray_LookupSpecial(obj, "__array_function__"); + PyObject *array_function = PyArray_LookupSpecial(obj, "__array_function__"); if (array_function == NULL && PyErr_Occurred()) { PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ } @@ -53,9 +52,7 @@ get_array_function(PyObject *obj) static void pyobject_array_insert(PyObject **array, int length, int index, PyObject *item) { - int j; - - for (j = length; j > index; j--) { + for (int j = length; j > index; j--) { array[j] = array[j - 1]; } array[index] = item; @@ -74,18 +71,16 @@ get_implementing_args_and_methods(PyObject *relevant_args, PyObject **methods) { int num_implementing_args = 0; - Py_ssize_t i; - int j; PyObject **items = PySequence_Fast_ITEMS(relevant_args); Py_ssize_t length = PySequence_Fast_GET_SIZE(relevant_args); - for (i = 0; i < length; i++) { + for (Py_ssize_t i = 0; i < length; i++) { int new_class = 1; PyObject *argument = items[i]; /* Have we seen this type before? */ - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { if (Py_TYPE(argument) == Py_TYPE(implementing_args[j])) { new_class = 0; break; @@ -109,7 +104,7 @@ get_implementing_args_and_methods(PyObject *relevant_args, /* "subclasses before superclasses, otherwise left to right" */ arg_index = num_implementing_args; - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { PyObject *other_type; other_type = (PyObject *)Py_TYPE(implementing_args[j]); if (PyObject_IsInstance(argument, other_type)) { @@ -129,7 +124,7 @@ get_implementing_args_and_methods(PyObject *relevant_args, return num_implementing_args; fail: - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { Py_DECREF(implementing_args[j]); Py_DECREF(methods[j]); } @@ -161,13 +156,10 @@ NPY_NO_EXPORT PyObject * array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, PyObject *kwargs) { - Py_ssize_t j; - PyObject *implementation, *result; - PyObject **items = PySequence_Fast_ITEMS(types); Py_ssize_t length = PySequence_Fast_GET_SIZE(types); - for (j = 0; j < length; j++) { + for (Py_ssize_t j = 0; j < length; j++) { int is_subclass = PyObject_IsSubclass( items[j], (PyObject *)&PyArray_Type); if (is_subclass == -1) { @@ -179,11 +171,11 @@ array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, } } - implementation = PyObject_GetAttr(func, npy_ma_str_implementation); + PyObject *implementation = PyObject_GetAttr(func, npy_ma_str_implementation); if (implementation == NULL) { return NULL; } - result = PyObject_Call(implementation, args, kwargs); + PyObject *result = PyObject_Call(implementation, args, kwargs); Py_DECREF(implementation); return result; } @@ -208,32 +200,32 @@ call_array_function(PyObject* argument, PyObject* method, } -/* - * Implements the __array_function__ protocol for a function, as described in - * in NEP-18. See numpy.core.overrides for a full docstring. +/** + * Internal handler for the array-function dispatching. The helper returns + * either the result, or NotImplemented (as a borrowed reference). + * + * @param public_api The public API symbol used for dispatching + * @param relevant_args Arguments which may implement __array_function__ + * @param args Original arguments + * @param kwargs Original keyword arguments + * + * @returns The result of the dispatched version, or a borrowed reference + * to NotImplemented to indicate the default implementation should + * be used. */ NPY_NO_EXPORT PyObject * -array_implement_array_function( - PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +array_implement_array_function_internal( + PyObject *public_api, PyObject *relevant_args, + PyObject *args, PyObject *kwargs) { - PyObject *implementation, *public_api, *relevant_args, *args, *kwargs; - - PyObject *types = NULL; PyObject *implementing_args[NPY_MAXARGS]; PyObject *array_function_methods[NPY_MAXARGS]; + PyObject *types = NULL; - int j, any_overrides; - int num_implementing_args = 0; PyObject *result = NULL; static PyObject *errmsg_formatter = NULL; - if (!PyArg_UnpackTuple( - positional_args, "implement_array_function", 5, 5, - &implementation, &public_api, &relevant_args, &args, &kwargs)) { - return NULL; - } - relevant_args = PySequence_Fast( relevant_args, "dispatcher for __array_function__ did not return an iterable"); @@ -242,7 +234,7 @@ array_implement_array_function( } /* Collect __array_function__ implementations */ - num_implementing_args = get_implementing_args_and_methods( + int num_implementing_args = get_implementing_args_and_methods( relevant_args, implementing_args, array_function_methods); if (num_implementing_args == -1) { goto cleanup; @@ -254,15 +246,19 @@ array_implement_array_function( * arguments implement __array_function__ at all (e.g., if they are all * built-in types). */ - any_overrides = 0; - for (j = 0; j < num_implementing_args; j++) { + int any_overrides = 0; + for (int j = 0; j < num_implementing_args; j++) { if (!is_default_array_function(array_function_methods[j])) { any_overrides = 1; break; } } if (!any_overrides) { - result = PyObject_Call(implementation, args, kwargs); + /* + * When the default implementation should be called, return + * `Py_NotImplemented` to indicate this. + */ + result = Py_NotImplemented; goto cleanup; } @@ -275,14 +271,14 @@ array_implement_array_function( if (types == NULL) { goto cleanup; } - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { PyObject *arg_type = (PyObject *)Py_TYPE(implementing_args[j]); Py_INCREF(arg_type); PyTuple_SET_ITEM(types, j, arg_type); } /* Call __array_function__ methods */ - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { PyObject *argument = implementing_args[j]; PyObject *method = array_function_methods[j]; @@ -319,7 +315,7 @@ array_implement_array_function( } cleanup: - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { Py_DECREF(implementing_args[j]); Py_DECREF(array_function_methods[j]); } @@ -329,6 +325,92 @@ cleanup: } +/* + * Implements the __array_function__ protocol for a Python function, as described in + * in NEP-18. See numpy.core.overrides for a full docstring. + */ +NPY_NO_EXPORT PyObject * +array_implement_array_function( + PyObject *NPY_UNUSED(dummy), PyObject *positional_args) +{ + PyObject *implementation, *public_api, *relevant_args, *args, *kwargs; + + if (!PyArg_UnpackTuple( + positional_args, "implement_array_function", 5, 5, + &implementation, &public_api, &relevant_args, &args, &kwargs)) { + return NULL; + } + + /* Remove `like=` kwarg, which is NumPy-exclusive and thus not present + * in downstream libraries. + */ + if (kwargs != NULL && PyDict_Contains(kwargs, npy_ma_str_like)) { + PyDict_DelItem(kwargs, npy_ma_str_like); + } + + PyObject *res = array_implement_array_function_internal( + public_api, relevant_args, args, kwargs); + + if (res == Py_NotImplemented) { + return PyObject_Call(implementation, args, kwargs); + } + return res; +} + + +/* + * Implements the __array_function__ protocol for C array creation functions + * only. Added as an extension to NEP-18 in an effort to bring NEP-35 to + * life with minimal dispatch overhead. + */ +NPY_NO_EXPORT PyObject * +array_implement_c_array_function_creation( + const char *function_name, PyObject *args, PyObject *kwargs) +{ + if (kwargs == NULL) { + return Py_NotImplemented; + } + + /* Remove `like=` kwarg, which is NumPy-exclusive and thus not present + * in downstream libraries. If that key isn't present, return NULL and + * let originating call to continue. + */ + if (!PyDict_Contains(kwargs, npy_ma_str_like)) { + return Py_NotImplemented; + } + + PyObject *relevant_args = PyTuple_Pack(1, + PyDict_GetItem(kwargs, npy_ma_str_like)); + if (relevant_args == NULL) { + return NULL; + } + PyDict_DelItem(kwargs, npy_ma_str_like); + + PyObject *numpy_module = PyImport_Import(npy_ma_str_numpy); + if (numpy_module == NULL) { + return NULL; + } + + PyObject *public_api = PyObject_GetAttrString(numpy_module, function_name); + Py_DECREF(numpy_module); + if (public_api == NULL) { + return NULL; + } + if (!PyCallable_Check(public_api)) { + Py_DECREF(public_api); + return PyErr_Format(PyExc_RuntimeError, + "numpy.%s is not callable.", + function_name); + } + + PyObject* result = array_implement_array_function_internal( + public_api, relevant_args, args, kwargs); + + Py_DECREF(public_api); + return result; +} + + /* * Python wrapper for get_implementing_args_and_methods, for testing purposes. */ @@ -337,8 +419,6 @@ array__get_implementing_args( PyObject *NPY_UNUSED(dummy), PyObject *positional_args) { PyObject *relevant_args; - int j; - int num_implementing_args = 0; PyObject *implementing_args[NPY_MAXARGS]; PyObject *array_function_methods[NPY_MAXARGS]; PyObject *result = NULL; @@ -355,7 +435,7 @@ array__get_implementing_args( return NULL; } - num_implementing_args = get_implementing_args_and_methods( + int num_implementing_args = get_implementing_args_and_methods( relevant_args, implementing_args, array_function_methods); if (num_implementing_args == -1) { goto cleanup; @@ -366,14 +446,14 @@ array__get_implementing_args( if (result == NULL) { goto cleanup; } - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { PyObject *argument = implementing_args[j]; Py_INCREF(argument); PyList_SET_ITEM(result, j, argument); } cleanup: - for (j = 0; j < num_implementing_args; j++) { + for (int j = 0; j < num_implementing_args; j++) { Py_DECREF(implementing_args[j]); Py_DECREF(array_function_methods[j]); } diff --git a/numpy/core/src/multiarray/arrayfunction_override.h b/numpy/core/src/multiarray/arrayfunction_override.h index 0d224e2b6..fdcf1746d 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.h +++ b/numpy/core/src/multiarray/arrayfunction_override.h @@ -9,6 +9,10 @@ NPY_NO_EXPORT PyObject * array__get_implementing_args( PyObject *NPY_UNUSED(dummy), PyObject *positional_args); +NPY_NO_EXPORT PyObject * +array_implement_c_array_function_creation( + const char *function_name, PyObject *args, PyObject *kwargs); + NPY_NO_EXPORT PyObject * array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, PyObject *kwargs); diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index c79d9a845..bc367b78d 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1582,13 +1582,16 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) npy_bool subok = NPY_FALSE; npy_bool copy = NPY_TRUE; int ndmin = 0, nd; + PyObject* like; PyArray_Descr *type = NULL; PyArray_Descr *oldtype = NULL; NPY_ORDER order = NPY_KEEPORDER; int flags = 0; - static char *kwd[]= {"object", "dtype", "copy", "order", "subok", - "ndmin", NULL}; + PyObject* array_function_result = NULL; + + static char *kwd[] = {"object", "dtype", "copy", "order", "subok", + "ndmin", "like", NULL}; if (PyTuple_GET_SIZE(args) > 2) { PyErr_Format(PyExc_TypeError, @@ -1597,6 +1600,12 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) return NULL; } + array_function_result = array_implement_c_array_function_creation( + "array", args, kws); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + /* super-fast path for ndarray argument calls */ if (PyTuple_GET_SIZE(args) == 0) { goto full_path; @@ -1674,13 +1683,14 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) } full_path: - if (!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i:array", kwd, + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i$O:array", kwd, &op, PyArray_DescrConverter2, &type, PyArray_BoolConverter, ©, PyArray_OrderConverter, &order, PyArray_BoolConverter, &subok, - &ndmin)) { + &ndmin, + &like)) { goto clean_type; } @@ -1817,20 +1827,29 @@ static PyObject * array_empty(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"shape", "dtype", "order", NULL}; + static char *kwlist[] = {"shape", "dtype", "order", "like", NULL}; PyArray_Descr *typecode = NULL; PyArray_Dims shape = {NULL, 0}; NPY_ORDER order = NPY_CORDER; + PyObject *like = NULL; npy_bool is_f_order; + PyObject *array_function_result = NULL; PyArrayObject *ret = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&:empty", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&$O:empty", kwlist, PyArray_IntpConverter, &shape, PyArray_DescrConverter, &typecode, - PyArray_OrderConverter, &order)) { + PyArray_OrderConverter, &order, + &like)) { goto fail; } + array_function_result = array_implement_c_array_function_creation( + "empty", args, kwds); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + switch (order) { case NPY_CORDER: is_f_order = NPY_FALSE; @@ -1984,20 +2003,29 @@ array_scalar(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) static PyObject * array_zeros(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"shape", "dtype", "order", NULL}; + static char *kwlist[] = {"shape", "dtype", "order", "like", NULL}; PyArray_Descr *typecode = NULL; PyArray_Dims shape = {NULL, 0}; NPY_ORDER order = NPY_CORDER; + PyObject *like = NULL; npy_bool is_f_order = NPY_FALSE; + PyObject *array_function_result = NULL; PyArrayObject *ret = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&:zeros", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&$O:zeros", kwlist, PyArray_IntpConverter, &shape, PyArray_DescrConverter, &typecode, - PyArray_OrderConverter, &order)) { + PyArray_OrderConverter, &order, + &like)) { goto fail; } + array_function_result = array_implement_c_array_function_creation( + "zeros", args, kwds); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + switch (order) { case NPY_CORDER: is_f_order = NPY_FALSE; @@ -2050,16 +2078,24 @@ array_fromstring(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds Py_ssize_t nin = -1; char *sep = NULL; Py_ssize_t s; - static char *kwlist[] = {"string", "dtype", "count", "sep", NULL}; + static char *kwlist[] = {"string", "dtype", "count", "sep", "like", NULL}; + PyObject *like = NULL; PyArray_Descr *descr = NULL; + PyObject *array_function_result = NULL; if (!PyArg_ParseTupleAndKeywords(args, keywds, - "s#|O&" NPY_SSIZE_T_PYFMT "s:fromstring", kwlist, - &data, &s, PyArray_DescrConverter, &descr, &nin, &sep)) { + "s#|O&" NPY_SSIZE_T_PYFMT "s$O:fromstring", kwlist, + &data, &s, PyArray_DescrConverter, &descr, &nin, &sep, &like)) { Py_XDECREF(descr); return NULL; } + array_function_result = array_implement_c_array_function_creation( + "fromstring", args, keywds); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + /* binary mode, condition copied from PyArray_FromString */ if (sep == NULL || strlen(sep) == 0) { /* Numpy 1.14, 2017-10-19 */ @@ -2082,19 +2118,27 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; char *sep = ""; Py_ssize_t nin = -1; - static char *kwlist[] = {"file", "dtype", "count", "sep", "offset", NULL}; + static char *kwlist[] = {"file", "dtype", "count", "sep", "offset", "like", NULL}; + PyObject *like = NULL; PyArray_Descr *type = NULL; + PyObject *array_function_result = NULL; int own; npy_off_t orig_pos = 0, offset = 0; FILE *fp; if (!PyArg_ParseTupleAndKeywords(args, keywds, - "O|O&" NPY_SSIZE_T_PYFMT "s" NPY_OFF_T_PYFMT ":fromfile", kwlist, - &file, PyArray_DescrConverter, &type, &nin, &sep, &offset)) { + "O|O&" NPY_SSIZE_T_PYFMT "s" NPY_OFF_T_PYFMT "$O:fromfile", kwlist, + &file, PyArray_DescrConverter, &type, &nin, &sep, &offset, &like)) { Py_XDECREF(type); return NULL; } + array_function_result = array_implement_c_array_function_creation( + "fromfile", args, keywds); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + file = NpyPath_PathlikeToFspath(file); if (file == NULL) { return NULL; @@ -2161,15 +2205,24 @@ array_fromiter(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) { PyObject *iter; Py_ssize_t nin = -1; - static char *kwlist[] = {"iter", "dtype", "count", NULL}; + static char *kwlist[] = {"iter", "dtype", "count", "like", NULL}; + PyObject *like = NULL; PyArray_Descr *descr = NULL; + PyObject *array_function_result = NULL; if (!PyArg_ParseTupleAndKeywords(args, keywds, - "OO&|" NPY_SSIZE_T_PYFMT ":fromiter", kwlist, - &iter, PyArray_DescrConverter, &descr, &nin)) { + "OO&|" NPY_SSIZE_T_PYFMT "$O:fromiter", kwlist, + &iter, PyArray_DescrConverter, &descr, &nin, &like)) { Py_XDECREF(descr); return NULL; } + + array_function_result = array_implement_c_array_function_creation( + "fromiter", args, keywds); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + return PyArray_FromIter(iter, descr, (npy_intp)nin); } @@ -2178,15 +2231,24 @@ array_frombuffer(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds { PyObject *obj = NULL; Py_ssize_t nin = -1, offset = 0; - static char *kwlist[] = {"buffer", "dtype", "count", "offset", NULL}; + static char *kwlist[] = {"buffer", "dtype", "count", "offset", "like", NULL}; + PyObject *like = NULL; PyArray_Descr *type = NULL; + PyObject *array_function_result = NULL; if (!PyArg_ParseTupleAndKeywords(args, keywds, - "O|O&" NPY_SSIZE_T_PYFMT NPY_SSIZE_T_PYFMT ":frombuffer", kwlist, - &obj, PyArray_DescrConverter, &type, &nin, &offset)) { + "O|O&" NPY_SSIZE_T_PYFMT NPY_SSIZE_T_PYFMT "$O:frombuffer", kwlist, + &obj, PyArray_DescrConverter, &type, &nin, &offset, &like)) { Py_XDECREF(type); return NULL; } + + array_function_result = array_implement_c_array_function_creation( + "frombuffer", args, keywds); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + if (type == NULL) { type = PyArray_DescrFromType(NPY_DEFAULT_TYPE); } @@ -2766,17 +2828,27 @@ array_correlate2(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) static PyObject * array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { PyObject *o_start = NULL, *o_stop = NULL, *o_step = NULL, *range=NULL; - static char *kwd[]= {"start", "stop", "step", "dtype", NULL}; + PyObject *like = NULL; + PyObject *array_function_result = NULL; + static char *kwd[] = {"start", "stop", "step", "dtype", "like", NULL}; PyArray_Descr *typecode = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&:arange", kwd, + if (!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&$O:arange", kwd, &o_start, &o_stop, &o_step, - PyArray_DescrConverter2, &typecode)) { + PyArray_DescrConverter2, &typecode, + &like)) { Py_XDECREF(typecode); return NULL; } + + array_function_result = array_implement_c_array_function_creation( + "arange", args, kws); + if (array_function_result != Py_NotImplemented) { + return array_function_result; + } + range = PyArray_ArangeObj(o_start, o_stop, o_step, typecode); Py_XDECREF(typecode); @@ -4331,6 +4403,8 @@ NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_dtype = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_ndmin = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_axis1 = NULL; NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_axis2 = NULL; +NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_like = NULL; +NPY_VISIBILITY_HIDDEN PyObject * npy_ma_str_numpy = NULL; static int intern_strings(void) @@ -4347,12 +4421,15 @@ intern_strings(void) npy_ma_str_ndmin = PyUnicode_InternFromString("ndmin"); npy_ma_str_axis1 = PyUnicode_InternFromString("axis1"); npy_ma_str_axis2 = PyUnicode_InternFromString("axis2"); + npy_ma_str_like = PyUnicode_InternFromString("like"); + npy_ma_str_numpy = PyUnicode_InternFromString("numpy"); return npy_ma_str_array && npy_ma_str_array_prepare && npy_ma_str_array_wrap && npy_ma_str_array_finalize && npy_ma_str_ufunc && npy_ma_str_implementation && npy_ma_str_order && npy_ma_str_copy && npy_ma_str_dtype && - npy_ma_str_ndmin && npy_ma_str_axis1 && npy_ma_str_axis2; + npy_ma_str_ndmin && npy_ma_str_axis1 && npy_ma_str_axis2 && + npy_ma_str_like && npy_ma_str_numpy; } static struct PyModuleDef moduledef = { diff --git a/numpy/core/src/multiarray/multiarraymodule.h b/numpy/core/src/multiarray/multiarraymodule.h index dd437e091..d3ee3337c 100644 --- a/numpy/core/src/multiarray/multiarraymodule.h +++ b/numpy/core/src/multiarray/multiarraymodule.h @@ -13,5 +13,7 @@ NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_dtype; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_ndmin; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_axis1; NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_axis2; +NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_like; +NPY_VISIBILITY_HIDDEN extern PyObject * npy_ma_str_numpy; #endif diff --git a/numpy/core/tests/test_overrides.py b/numpy/core/tests/test_overrides.py index 7e73d8c03..42600a12b 100644 --- a/numpy/core/tests/test_overrides.py +++ b/numpy/core/tests/test_overrides.py @@ -1,5 +1,7 @@ import inspect import sys +import tempfile +from io import StringIO from unittest import mock import numpy as np @@ -425,3 +427,168 @@ class TestNumPyFunctions: # note: the internal implementation of np.sum() calls the .sum() method array = np.array(1).view(MyArray) assert_equal(np.sum(array), 'summed') + + +class TestArrayLike: + + class MyArray(): + + def __init__(self, function=None): + self.function = function + + def __array_function__(self, func, types, args, kwargs): + try: + my_func = getattr(TestArrayLike.MyArray, func.__name__) + except AttributeError: + return NotImplemented + return my_func(*args, **kwargs) + + class MyNoArrayFunctionArray(): + + def __init__(self, function=None): + self.function = function + + def add_method(name, arr_class, enable_value_error=False): + def _definition(*args, **kwargs): + # Check that `like=` isn't propagated downstream + assert 'like' not in kwargs + + if enable_value_error and 'value_error' in kwargs: + raise ValueError + + return arr_class(getattr(arr_class, name)) + setattr(arr_class, name, _definition) + + def func_args(*args, **kwargs): + return args, kwargs + + @requires_array_function + def test_array_like_not_implemented(self): + TestArrayLike.add_method('array', TestArrayLike.MyArray) + + ref = TestArrayLike.MyArray.array() + + with assert_raises_regex(TypeError, 'no implementation found'): + array_like = np.asarray(1, like=ref) + + _array_tests = [ + ('array', *func_args((1,))), + ('asarray', *func_args((1,))), + ('asanyarray', *func_args((1,))), + ('ascontiguousarray', *func_args((2, 3))), + ('asfortranarray', *func_args((2, 3))), + ('require', *func_args((np.arange(6).reshape(2, 3),), + requirements=['A', 'F'])), + ('empty', *func_args((1,))), + ('full', *func_args((1,), 2)), + ('ones', *func_args((1,))), + ('zeros', *func_args((1,))), + ('arange', *func_args(3)), + ('frombuffer', *func_args(b'\x00' * 8, dtype=int)), + ('fromiter', *func_args(range(3), dtype=int)), + ('fromstring', *func_args('1,2', dtype=int, sep=',')), + ('loadtxt', *func_args(lambda: StringIO('0 1\n2 3'))), + ('genfromtxt', *func_args(lambda: StringIO(u'1,2.1'), + dtype=[('int', 'i8'), ('float', 'f8')], + delimiter=',')), + ] + + @pytest.mark.parametrize('function, args, kwargs', _array_tests) + @pytest.mark.parametrize('numpy_ref', [True, False]) + @requires_array_function + def test_array_like(self, function, args, kwargs, numpy_ref): + TestArrayLike.add_method('array', TestArrayLike.MyArray) + TestArrayLike.add_method(function, TestArrayLike.MyArray) + np_func = getattr(np, function) + my_func = getattr(TestArrayLike.MyArray, function) + + if numpy_ref is True: + ref = np.array(1) + else: + ref = TestArrayLike.MyArray.array() + + like_args = tuple(a() if callable(a) else a for a in args) + array_like = np_func(*like_args, **kwargs, like=ref) + + if numpy_ref is True: + assert type(array_like) is np.ndarray + + np_args = tuple(a() if callable(a) else a for a in args) + np_arr = np_func(*np_args, **kwargs) + + # Special-case np.empty to ensure values match + if function == "empty": + np_arr.fill(1) + array_like.fill(1) + + assert_equal(array_like, np_arr) + else: + assert type(array_like) is TestArrayLike.MyArray + assert array_like.function is my_func + + @pytest.mark.parametrize('function, args, kwargs', _array_tests) + @pytest.mark.parametrize('numpy_ref', [True, False]) + @requires_array_function + def test_no_array_function_like(self, function, args, kwargs, numpy_ref): + TestArrayLike.add_method('array', TestArrayLike.MyNoArrayFunctionArray) + TestArrayLike.add_method(function, TestArrayLike.MyNoArrayFunctionArray) + np_func = getattr(np, function) + my_func = getattr(TestArrayLike.MyNoArrayFunctionArray, function) + + if numpy_ref is True: + ref = np.array(1) + else: + ref = TestArrayLike.MyNoArrayFunctionArray.array() + + like_args = tuple(a() if callable(a) else a for a in args) + array_like = np_func(*like_args, **kwargs, like=ref) + + assert type(array_like) is np.ndarray + if numpy_ref is True: + np_args = tuple(a() if callable(a) else a for a in args) + np_arr = np_func(*np_args, **kwargs) + + # Special-case np.empty to ensure values match + if function == "empty": + np_arr.fill(1) + array_like.fill(1) + + assert_equal(array_like, np_arr) + + @pytest.mark.parametrize('numpy_ref', [True, False]) + def test_array_like_fromfile(self, numpy_ref): + TestArrayLike.add_method('array', TestArrayLike.MyArray) + TestArrayLike.add_method("fromfile", TestArrayLike.MyArray) + + if numpy_ref is True: + ref = np.array(1) + else: + ref = TestArrayLike.MyArray.array() + + data = np.random.random(5) + + fname = tempfile.mkstemp()[1] + data.tofile(fname) + + array_like = np.fromfile(fname, like=ref) + if numpy_ref is True: + assert type(array_like) is np.ndarray + np_res = np.fromfile(fname, like=ref) + assert_equal(np_res, data) + assert_equal(array_like, np_res) + else: + assert type(array_like) is TestArrayLike.MyArray + assert array_like.function is TestArrayLike.MyArray.fromfile + + @requires_array_function + def test_exception_handling(self): + TestArrayLike.add_method( + 'array', + TestArrayLike.MyArray, + enable_value_error=True, + ) + + ref = TestArrayLike.MyArray.array() + + with assert_raises(ValueError): + np.array(1, value_error=True, like=ref) diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 58affc2fc..cc3465cc6 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -14,7 +14,7 @@ from . import format from ._datasource import DataSource from numpy.core import overrides from numpy.core.multiarray import packbits, unpackbits -from numpy.core.overrides import set_module +from numpy.core.overrides import set_array_function_like_doc, set_module from numpy.core._internal import recursive from ._iotools import ( LineSplitter, NameValidator, StringConverter, ConverterError, @@ -790,10 +790,17 @@ def _getconv(dtype): _loadtxt_chunksize = 50000 +def _loadtxt_dispatcher(fname, dtype=None, comments=None, delimiter=None, + converters=None, skiprows=None, usecols=None, unpack=None, + ndmin=None, encoding=None, max_rows=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') def loadtxt(fname, dtype=float, comments='#', delimiter=None, converters=None, skiprows=0, usecols=None, unpack=False, - ndmin=0, encoding='bytes', max_rows=None): + ndmin=0, encoding='bytes', max_rows=None, *, like=None): r""" Load data from a text file. @@ -860,6 +867,9 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, is to read all the lines. .. versionadded:: 1.16.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -917,6 +927,14 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, [-17.57, 63.94]]) """ + if like is not None: + return _loadtxt_with_like( + fname, dtype=dtype, comments=comments, delimiter=delimiter, + converters=converters, skiprows=skiprows, usecols=usecols, + unpack=unpack, ndmin=ndmin, encoding=encoding, + max_rows=max_rows, like=like + ) + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Nested functions used by loadtxt. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1201,6 +1219,11 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None, return X +_loadtxt_with_like = array_function_dispatch( + _loadtxt_dispatcher +)(loadtxt) + + def _savetxt_dispatcher(fname, X, fmt=None, delimiter=None, newline=None, header=None, footer=None, comments=None, encoding=None): @@ -1554,6 +1577,18 @@ def fromregex(file, regexp, dtype, encoding=None): #####-------------------------------------------------------------------------- +def _genfromtxt_dispatcher(fname, dtype=None, comments=None, delimiter=None, + skip_header=None, skip_footer=None, converters=None, + missing_values=None, filling_values=None, usecols=None, + names=None, excludelist=None, deletechars=None, + replace_space=None, autostrip=None, case_sensitive=None, + defaultfmt=None, unpack=None, usemask=None, loose=None, + invalid_raise=None, max_rows=None, encoding=None, *, + like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') def genfromtxt(fname, dtype=float, comments='#', delimiter=None, skip_header=0, skip_footer=0, converters=None, @@ -1562,7 +1597,8 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, deletechars=''.join(sorted(NameValidator.defaultdeletechars)), replace_space='_', autostrip=False, case_sensitive=True, defaultfmt="f%i", unpack=None, usemask=False, loose=True, - invalid_raise=True, max_rows=None, encoding='bytes'): + invalid_raise=True, max_rows=None, encoding='bytes', *, + like=None): """ Load data from a text file, with missing values handled as specified. @@ -1659,6 +1695,9 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, to None the system default is used. The default value is 'bytes'. .. versionadded:: 1.14.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -1737,6 +1776,21 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, dtype=[('f0', 'S12'), ('f1', 'S12')]) """ + + if like is not None: + return _genfromtxt_with_like( + fname, dtype=dtype, comments=comments, delimiter=delimiter, + skip_header=skip_header, skip_footer=skip_footer, + converters=converters, missing_values=missing_values, + filling_values=filling_values, usecols=usecols, names=names, + excludelist=excludelist, deletechars=deletechars, + replace_space=replace_space, autostrip=autostrip, + case_sensitive=case_sensitive, defaultfmt=defaultfmt, + unpack=unpack, usemask=usemask, loose=loose, + invalid_raise=invalid_raise, max_rows=max_rows, encoding=encoding, + like=like + ) + if max_rows is not None: if skip_footer: raise ValueError( @@ -2250,6 +2304,11 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, return output.squeeze() +_genfromtxt_with_like = array_function_dispatch( + _genfromtxt_dispatcher +)(genfromtxt) + + def ndfromtxt(fname, **kwargs): """ Load ASCII data stored in a file and return it as a single array. diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index cd7484241..2b4cbdfbb 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -8,7 +8,7 @@ from numpy.core.numeric import ( asarray, where, int8, int16, int32, int64, empty, promote_types, diagonal, nonzero ) -from numpy.core.overrides import set_module +from numpy.core.overrides import set_array_function_like_doc, set_module from numpy.core import overrides from numpy.core import iinfo @@ -149,8 +149,13 @@ def flipud(m): return m[::-1, ...] +def _eye_dispatcher(N, M=None, k=None, dtype=None, order=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def eye(N, M=None, k=0, dtype=float, order='C'): +def eye(N, M=None, k=0, dtype=float, order='C', *, like=None): """ Return a 2-D array with ones on the diagonal and zeros elsewhere. @@ -171,6 +176,9 @@ def eye(N, M=None, k=0, dtype=float, order='C'): column-major (Fortran-style) order in memory. .. versionadded:: 1.14.0 + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -194,6 +202,8 @@ def eye(N, M=None, k=0, dtype=float, order='C'): [0., 0., 0.]]) """ + if like is not None: + return _eye_with_like(N, M=M, k=k, dtype=dtype, order=order, like=like) if M is None: M = N m = zeros((N, M), dtype=dtype, order=order) @@ -207,6 +217,11 @@ def eye(N, M=None, k=0, dtype=float, order='C'): return m +_eye_with_like = array_function_dispatch( + _eye_dispatcher +)(eye) + + def _diag_dispatcher(v, k=None): return (v,) @@ -343,8 +358,13 @@ def diagflat(v, k=0): return wrap(res) +def _tri_dispatcher(N, M=None, k=None, dtype=None, *, like=None): + return (like,) + + +@set_array_function_like_doc @set_module('numpy') -def tri(N, M=None, k=0, dtype=float): +def tri(N, M=None, k=0, dtype=float, *, like=None): """ An array with ones at and below the given diagonal and zeros elsewhere. @@ -361,6 +381,9 @@ def tri(N, M=None, k=0, dtype=float): and `k` > 0 is above. The default is 0. dtype : dtype, optional Data type of the returned array. The default is float. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 Returns ------- @@ -381,6 +404,9 @@ def tri(N, M=None, k=0, dtype=float): [1., 1., 0., 0., 0.]]) """ + if like is not None: + return _tri_with_like(N, M=M, k=k, dtype=dtype, like=like) + if M is None: M = N @@ -393,6 +419,11 @@ def tri(N, M=None, k=0, dtype=float): return m +_tri_with_like = array_function_dispatch( + _tri_dispatcher +)(tri) + + def _trilu_dispatcher(m, k=None): return (m,) -- cgit v1.2.1 From a6c974489b753437dc5929326d4c320d39f26c08 Mon Sep 17 00:00:00 2001 From: Ryan Soklaski Date: Fri, 28 Aug 2020 18:52:37 -0400 Subject: DOC: Correct error in description of ndarray.base Found in subsection of Subclassing ndarray in python: Extra gotchas - custom __del__ methods and ndarray.base --- numpy/doc/subclassing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/doc/subclassing.py b/numpy/doc/subclassing.py index 7dc10e1c8..d6a4a765b 100644 --- a/numpy/doc/subclassing.py +++ b/numpy/doc/subclassing.py @@ -685,8 +685,8 @@ True True >>> # Take a view of a view >>> v2 = v1[1:] ->>> # base points to the view it derived from ->>> v2.base is v1 +>>> # base points to the original array that it was derived from +>>> v2.base is arr True In general, if the array owns its own memory, as for ``arr`` in this -- cgit v1.2.1 From df85fb95cfece479ec06357c01753c0c0b670317 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 25 Aug 2020 18:19:33 -0500 Subject: TST: Add tests mapping out the rules for metadata in promotion Note that these rules do not actually make much sense often. They depend on a lot of suble branches, which probably grew over time without much regard for metadata. The test is ugly, but maps them out to detect changes in behaviour. At some point, we should probably define how metadata and promotion works. Such as always merging dicts, always drop if the original dtype is lost? That assumes that metadata can be a useful concept which makes sense to "inherit" during promotion. Its also plausible that we should just always strip metadata during promotion. --- numpy/core/tests/test_numeric.py | 104 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index 0b27c54dd..ae5ee4c88 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -922,6 +922,110 @@ class TestTypes: assert_equal(np.promote_types('u8', 'S1'), np.dtype('S20')) assert_equal(np.promote_types('u8', 'S30'), np.dtype('S30')) + @pytest.mark.parametrize("dtype", + list(np.typecodes["All"]) + + ["i,i", "S3", "S100", "U3", "U100", rational]) + def test_promote_identical_types_metadata(self, dtype): + # The same type passed in twice to promote types always + # preserves metadata + metadata = {1: 1} + dtype = np.dtype(dtype, metadata=metadata) + + res = np.promote_types(dtype, dtype) + assert res.metadata == dtype.metadata + + # byte-swapping preserves and makes the dtype native: + dtype = dtype.newbyteorder() + if dtype.isnative: + # The type does not have byte swapping + return + + res = np.promote_types(dtype, dtype) + if res.char in "?bhilqpBHILQPefdgFDGOmM": + # Metadata is lost for simple promotions (they create a new dtype) + assert res.metadata is None + else: + assert res.metadata == metadata + if dtype.kind != "V": + # the result is native (except for structured void) + assert res.isnative + + @pytest.mark.slow + @pytest.mark.parametrize(["dtype1", "dtype2"], + itertools.product( + list(np.typecodes["All"]) + + ["i,i", "S3", "S100", "U3", "U100", rational], + repeat=2)) + def test_promote_types_metadata(self, dtype1, dtype2): + """Metadata handling in promotion does not appear formalized + right now in NumPy. This test should thus be considered to + document behaviour, rather than test the correct definition of it. + + This test is very ugly, it was useful for rewriting part of the + promotion, but probably should eventually be replaced/deleted + (i.e. when metadata handling in promotion is better defined). + """ + metadata1 = {1: 1} + metadata2 = {2: 2} + dtype1 = np.dtype(dtype1, metadata=metadata1) + dtype2 = np.dtype(dtype2, metadata=metadata2) + + try: + res = np.promote_types(dtype1, dtype2) + except TypeError: + # Promotion failed, this test only checks metadata + return + + # The rules for when metadata is preserved and which dtypes metadta + # will be used are very confusing and depend on multiple paths. + # This long if statement attempts to reproduce this: + if dtype1.type is rational or dtype2.type is rational: + # User dtype promotion preserves byte-order here: + if np.can_cast(res, dtype1): + assert res.metadata == dtype1.metadata + else: + assert res.metadata == dtype2.metadata + + elif res.char in "?bhilqpBHILQPefdgFDGOmM": + # All simple types lose metadata (due to using promotion table): + assert res.metadata is None + elif res.kind in "SU" and dtype1 == dtype2: + # Strings give precedence to the second dtype: + assert res is dtype2 + elif res == dtype1: + # If one result is the result, it is usually returned unchanged: + assert res is dtype1 + elif res == dtype2: + # If one result is the result, it is usually returned unchanged: + assert res is dtype2 + elif dtype1.kind == "S" and dtype2.kind == "U": + # Promotion creates a new unicode dtype from scratch + assert res.metadata is None + elif dtype1.kind == "U" and dtype2.kind == "S": + # Promotion creates a new unicode dtype from scratch + assert res.metadata is None + elif res.kind in "SU" and dtype2.kind != res.kind: + # We build on top of dtype1: + assert res.metadata == dtype1.metadata + elif res.kind in "SU" and res.kind == dtype1.kind: + assert res.metadata == dtype1.metadata + elif res.kind in "SU" and res.kind == dtype2.kind: + assert res.metadata == dtype2.metadata + else: + assert res.metadata is None + + # Try again for byteswapped version + dtype1 = dtype1.newbyteorder() + assert dtype1.metadata == metadata1 + res_bs = np.promote_types(dtype1, dtype2) + if res_bs.names is not None: + # Structured promotion doesn't remove byteswap: + assert res_bs.newbyteorder() == res + else: + assert res_bs == res + assert res_bs.metadata == res.metadata + + def test_can_cast(self): assert_(np.can_cast(np.int32, np.int64)) assert_(np.can_cast(np.float64, complex)) -- cgit v1.2.1 From 2104059250be3a8a434404692154151d82e569c0 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 28 Aug 2020 18:38:39 -0500 Subject: DOC: Document ``dtype.metadata`` This adds some basic documentation to the ``dtype.metadata`` attribute. Note that none of the documentation mentions metadata including https://numpy.org/devdocs/reference/arrays.dtypes.html --- doc/source/reference/arrays.dtypes.rst | 7 +++++++ numpy/core/_add_newdocs.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/doc/source/reference/arrays.dtypes.rst b/doc/source/reference/arrays.dtypes.rst index c7703764f..575984707 100644 --- a/doc/source/reference/arrays.dtypes.rst +++ b/doc/source/reference/arrays.dtypes.rst @@ -537,6 +537,13 @@ Attributes providing additional information: dtype.alignment dtype.base +Metadata attached by the user: + +.. autosummary:: + :toctree: generated/ + + dtype.metadata + Methods ------- diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index c3b4374f4..e32b32fbd 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -5491,6 +5491,39 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('kind', """)) +add_newdoc('numpy.core.multiarray', 'dtype', ('metadata', + """ + Either ``None`` or a readonly dictionary of metadata (mappingproxy). + + The metadata field can be set using any dictionary at data-type + creation. Note that whether or not operations on arrays with metadata + attached to their datatypes is currently not well defined and should + not be relied on. + + Examples + -------- + + >>> dt = np.dtype(float, metadata={"key": "value"}) + >>> dt.metadata["key"] + 'value' + >>> arr = np.array([1, 2, 3], dtype=dt) + >>> arr.dtype.metadata + mappingproxy({'key': 'value'}) + + Some operations may preserve metadata (identical data types): + + >>> (arr + arr).dtype.metadata + mappingproxy({'key': 'value'}) + + But for example, adding two arrays with different metadata does not + propagate either one: + + >>> dt2 = np.dtype(float, metadata={"key2": "value2"}) + >>> arr2 = np.array([3, 2, 1], dtype=dt2) + >>> (arr + arr2).dtype.metadata is None + True # The metadata field is cleared so None is returned + """)) + add_newdoc('numpy.core.multiarray', 'dtype', ('name', """ A bit-width name for this data-type. -- cgit v1.2.1 From 6caae1871882066d2d37d9feee7631ecbbe9a88a Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 14:31:50 +0200 Subject: ENH: Increase the use of `Literal` types --- numpy/__init__.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 17f7dda67..f11c4afcf 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -104,6 +104,7 @@ class dtype: @property def subdtype(self) -> Optional[Tuple[dtype, _Shape]]: ... def newbyteorder(self, new_order: str = ...) -> dtype: ... + def newbyteorder(self, new_order: _ByteOrder = ...) -> dtype: ... # Leave str and type for end to avoid having to use `builtins.str` # everywhere. See https://github.com/python/mypy/issues/3775 @property -- cgit v1.2.1 From b2b12b68978dfdd84243629e5ee989ae982bd699 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 14:32:26 +0200 Subject: ENH: Increase the use of `Literal` types --- numpy/__init__.pyi | 67 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index f11c4afcf..cee6b59b3 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -48,6 +48,7 @@ else: def __getattr__(name: str) -> Any: ... _NdArraySubClass = TypeVar("_NdArraySubClass", bound=ndarray) +_ByteOrder = Literal["S", "<", ">", "=", "|", "L", "B", "N", "I"] class dtype: names: Optional[Tuple[str, ...]] @@ -103,7 +104,6 @@ class dtype: def ndim(self) -> int: ... @property def subdtype(self) -> Optional[Tuple[dtype, _Shape]]: ... - def newbyteorder(self, new_order: str = ...) -> dtype: ... def newbyteorder(self, new_order: _ByteOrder = ...) -> dtype: ... # Leave str and type for end to avoid having to use `builtins.str` # everywhere. See https://github.com/python/mypy/issues/3775 @@ -157,6 +157,10 @@ class flatiter(Generic[_ArraySelf]): def __iter__(self: _FlatIterSelf) -> _FlatIterSelf: ... def __next__(self) -> generic: ... +_OrderKACF = Optional[Literal["K", "A", "C", "F"]] +_OrderACF = Optional[Literal["A", "C", "F"]] +_OrderCF = Optional[Literal["C", "F"]] + _ArraySelf = TypeVar("_ArraySelf", bound=_ArrayOrScalarCommon) class _ArrayOrScalarCommon( @@ -198,8 +202,8 @@ class _ArrayOrScalarCommon( def __bytes__(self) -> bytes: ... def __str__(self) -> str: ... def __repr__(self) -> str: ... - def __copy__(self: _ArraySelf, order: str = ...) -> _ArraySelf: ... - def __deepcopy__(self: _ArraySelf, memo: dict) -> _ArraySelf: ... + def __copy__(self: _ArraySelf) -> _ArraySelf: ... + def __deepcopy__(self: _ArraySelf, __memo: Optional[dict] = ...) -> _ArraySelf: ... def __lt__(self, other): ... def __le__(self, other): ... def __eq__(self, other): ... @@ -260,6 +264,7 @@ class _ArrayOrScalarCommon( def __getattr__(self, name) -> Any: ... _BufferType = Union[ndarray, bytes, bytearray, memoryview] +_Casting = Literal["no", "equiv", "safe", "same_kind", "unsafe"] class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): @property @@ -277,7 +282,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): buffer: _BufferType = ..., offset: int = ..., strides: _ShapeLike = ..., - order: Optional[str] = ..., + order: _OrderKACF = ..., ) -> _ArraySelf: ... @property def dtype(self) -> _Dtype: ... @@ -303,7 +308,7 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def itemset(self, __value: Any) -> None: ... @overload def itemset(self, __item: _ShapeLike, __value: Any) -> None: ... - def tobytes(self, order: Optional[str] = ...) -> bytes: ... + def tobytes(self, order: _OrderKACF = ...) -> bytes: ... def tofile( self, fid: Union[IO[bytes], str], sep: str = ..., format: str = ... ) -> None: ... @@ -312,13 +317,13 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): def astype( self: _ArraySelf, dtype: DtypeLike, - order: str = ..., - casting: str = ..., + order: _OrderKACF = ..., + casting: _Casting = ..., subok: bool = ..., copy: bool = ..., ) -> _ArraySelf: ... def byteswap(self: _ArraySelf, inplace: bool = ...) -> _ArraySelf: ... - def copy(self: _ArraySelf, order: str = ...) -> _ArraySelf: ... + def copy(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... @overload def view(self, type: Type[_NdArraySubClass]) -> _NdArraySubClass: ... @overload @@ -337,10 +342,12 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # Shape manipulation @overload def reshape( - self: _ArraySelf, shape: Sequence[int], *, order: str = ... + self: _ArraySelf, shape: Sequence[int], *, order: _OrderACF = ... ) -> _ArraySelf: ... @overload - def reshape(self: _ArraySelf, *shape: int, order: str = ...) -> _ArraySelf: ... + def reshape( + self: _ArraySelf, *shape: int, order: _OrderACF = ... + ) -> _ArraySelf: ... @overload def resize(self, new_shape: Sequence[int], *, refcheck: bool = ...) -> None: ... @overload @@ -350,8 +357,8 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): @overload def transpose(self: _ArraySelf, *axes: int) -> _ArraySelf: ... def swapaxes(self: _ArraySelf, axis1: int, axis2: int) -> _ArraySelf: ... - def flatten(self: _ArraySelf, order: str = ...) -> _ArraySelf: ... - def ravel(self: _ArraySelf, order: str = ...) -> _ArraySelf: ... + def flatten(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... + def ravel(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def squeeze( self: _ArraySelf, axis: Union[int, Tuple[int, ...]] = ... ) -> _ArraySelf: ... @@ -522,7 +529,7 @@ def array( dtype: DtypeLike = ..., *, copy: bool = ..., - order: Optional[str] = ..., + order: _OrderKACF = ..., subok: bool = ..., ndmin: int = ..., like: ArrayLike = ..., @@ -530,42 +537,42 @@ def array( def zeros( shape: _ShapeLike, dtype: DtypeLike = ..., - order: Optional[str] = ..., + order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... def ones( shape: _ShapeLike, dtype: DtypeLike = ..., - order: Optional[str] = ..., + order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... def empty( shape: _ShapeLike, dtype: DtypeLike = ..., - order: Optional[str] = ..., + order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... def zeros_like( a: ArrayLike, dtype: DtypeLike = ..., - order: str = ..., + order: _OrderKACF = ..., subok: bool = ..., shape: Optional[Union[int, Sequence[int]]] = ..., ) -> ndarray: ... def ones_like( a: ArrayLike, dtype: DtypeLike = ..., - order: str = ..., + order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... def empty_like( a: ArrayLike, dtype: DtypeLike = ..., - order: str = ..., + order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... @@ -573,7 +580,7 @@ def full( shape: _ShapeLike, fill_value: Any, dtype: DtypeLike = ..., - order: str = ..., + order: _OrderCF = ..., *, like: ArrayLike = ..., ) -> ndarray: ... @@ -581,7 +588,7 @@ def full_like( a: ArrayLike, fill_value: Any, dtype: DtypeLike = ..., - order: str = ..., + order: _OrderKACF = ..., subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... @@ -591,8 +598,11 @@ def count_nonzero( def isfortran(a: ndarray) -> bool: ... def argwhere(a: ArrayLike) -> ndarray: ... def flatnonzero(a: ArrayLike) -> ndarray: ... -def correlate(a: ArrayLike, v: ArrayLike, mode: str = ...) -> ndarray: ... -def convolve(a: ArrayLike, v: ArrayLike, mode: str = ...) -> ndarray: ... + +_CorrelateMode = Literal["valid", "same", "full"] + +def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... +def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... def outer(a: ArrayLike, b: ArrayLike, out: ndarray = ...) -> ndarray: ... def tensordot( a: ArrayLike, @@ -714,10 +724,8 @@ class ufunc: axes: List[Any] = ..., axis: int = ..., keepdims: bool = ..., - # TODO: make this precise when we can use Literal. - casting: str = ..., - # TODO: make this precise when we can use Literal. - order: Optional[str] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., dtype: DtypeLike = ..., subok: bool = ..., signature: Union[str, Tuple[str]] = ..., @@ -896,7 +904,6 @@ def find_common_type( # Functions from np.core.fromnumeric _Mode = Literal["raise", "wrap", "clip"] -_Order = Literal["C", "F", "A"] _PartitionKind = Literal["introselect"] _SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] _Side = Literal["left", "right"] @@ -978,7 +985,7 @@ def take( out: Optional[ndarray] = ..., mode: _Mode = ..., ) -> Union[_ScalarNumpy, ndarray]: ... -def reshape(a: ArrayLike, newshape: _ShapeLike, order: _Order = ...) -> ndarray: ... +def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... @overload def choose( a: _ScalarIntOrBool, @@ -1092,7 +1099,7 @@ def trace( dtype: DtypeLike = ..., out: Optional[ndarray] = ..., ) -> Union[number, ndarray]: ... -def ravel(a: ArrayLike, order: _Order = ...) -> ndarray: ... +def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... def nonzero(a: ArrayLike) -> Tuple[ndarray, ...]: ... def shape(a: ArrayLike) -> _Shape: ... def compress( -- cgit v1.2.1 From 1f315c7e7f2498c1e4864194fdf7f62b1af557b0 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 14:33:13 +0200 Subject: TST: Add more `Literal`-related tests --- numpy/tests/typing/pass/literal.py | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 numpy/tests/typing/pass/literal.py diff --git a/numpy/tests/typing/pass/literal.py b/numpy/tests/typing/pass/literal.py new file mode 100644 index 000000000..321ce3c2b --- /dev/null +++ b/numpy/tests/typing/pass/literal.py @@ -0,0 +1,43 @@ +from functools import partial +from typing import Callable, List, Tuple + +import pytest # type: ignore +import numpy as np + +AR = np.array(0) +AR.setflags(write=False) + +KACF = frozenset({None, "K", "A", "C", "F"}) +ACF = frozenset({None, "A", "C", "F"}) +CF = frozenset({None, "C", "F"}) + +order_list: List[Tuple[frozenset, Callable]] = [ + (KACF, partial(np.ndarray, 1)), + (KACF, AR.tobytes), + (KACF, partial(AR.astype, int)), + (KACF, AR.copy), + (ACF, partial(AR.reshape, 1)), + (KACF, AR.flatten), + (KACF, AR.ravel), + (KACF, partial(np.array, 1)), + (CF, partial(np.zeros, 1)), + (CF, partial(np.ones, 1)), + (CF, partial(np.empty, 1)), + (CF, partial(np.full, 1, 1)), + (KACF, partial(np.zeros_like, AR)), + (KACF, partial(np.ones_like, AR)), + (KACF, partial(np.empty_like, AR)), + (KACF, partial(np.full_like, AR, 1)), + (KACF, partial(np.add, 1, 1)), # i.e. np.ufunc.__call__ + (ACF, partial(np.reshape, AR, 1)), + (KACF, partial(np.ravel, AR)), +] + +for order_set, func in order_list: + for order in order_set: + func(order=order) + + invalid_orders = KACF - order_set + for order in invalid_orders: + with pytest.raises(ValueError): + func(order=order) -- cgit v1.2.1 From 2dd186f43ea6fd9d5d609a1bc753f62790fbd27f Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 14:38:12 +0200 Subject: MAINT: Changed the `new_order` parameter to positional-only --- numpy/__init__.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index cee6b59b3..8d65c23c4 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -104,7 +104,7 @@ class dtype: def ndim(self) -> int: ... @property def subdtype(self) -> Optional[Tuple[dtype, _Shape]]: ... - def newbyteorder(self, new_order: _ByteOrder = ...) -> dtype: ... + def newbyteorder(self, __new_order: _ByteOrder = ...) -> dtype: ... # Leave str and type for end to avoid having to use `builtins.str` # everywhere. See https://github.com/python/mypy/issues/3775 @property -- cgit v1.2.1 From 590fedf93c7bc400c04aa86a6e02c0d4661c3b84 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 14:47:17 +0200 Subject: DOC: Update the `newbyteorder` signature(s) to reflect its positional-only parameter --- numpy/core/_add_newdocs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index f11f008b2..6347c8396 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -3273,7 +3273,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('min', add_newdoc('numpy.core.multiarray', 'ndarray', ('newbyteorder', """ - arr.newbyteorder(new_order='S') + arr.newbyteorder(new_order='S', /) Return the array with the same data viewed with a different byte order. @@ -5697,7 +5697,7 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('type', add_newdoc('numpy.core.multiarray', 'dtype', ('newbyteorder', """ - newbyteorder(new_order='S') + newbyteorder(new_order='S', /) Return a new dtype with a different byte order. -- cgit v1.2.1 From ceca75708ad24cdce0bbee7ad8d82d02fa748d74 Mon Sep 17 00:00:00 2001 From: Bas van Beek Date: Fri, 31 Jul 2020 15:26:48 +0200 Subject: DOC: Update the `newbyteorder` signature(s) to reflect its positional-only parameter --- numpy/core/_add_newdocs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 6347c8396..5e8fb3729 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -6073,7 +6073,7 @@ add_newdoc('numpy.core.numerictypes', 'generic', add_newdoc('numpy.core.numerictypes', 'generic', ('newbyteorder', """ - newbyteorder(new_order='S') + newbyteorder(new_order='S', /) Return a new `dtype` with a different byte order. -- cgit v1.2.1 From 48de200a2ed9b6e5d6866f06dec07c90d381f859 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 29 Aug 2020 22:08:15 +0300 Subject: DOC: update theme to 0.3.2 --- doc_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc_requirements.txt b/doc_requirements.txt index 9861ba0a3..d2f6edb98 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -3,4 +3,4 @@ ipython scipy matplotlib pandas -pydata-sphinx-theme==0.3.1 +pydata-sphinx-theme==0.3.2 -- cgit v1.2.1 From 4e4458ac87931c3ef87f1281cf73e7ad1b1daa14 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Sat, 29 Aug 2020 14:39:38 -0500 Subject: Update numpy/core/_add_newdocs.py Co-authored-by: Eric Wieser --- numpy/core/_add_newdocs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index e32b32fbd..45c03f70d 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -5496,9 +5496,9 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('metadata', Either ``None`` or a readonly dictionary of metadata (mappingproxy). The metadata field can be set using any dictionary at data-type - creation. Note that whether or not operations on arrays with metadata - attached to their datatypes is currently not well defined and should - not be relied on. + creation. Note that whether operations on arrays with metadata + attached to their datatypes propagate that metadata is currently + not well defined and should not be relied on. Examples -------- -- cgit v1.2.1 From b0222e05b9b98395fcdab4e6ac7f5fed518ec387 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Fri, 21 Aug 2020 07:25:17 +0100 Subject: MAINT: Remove deplicated symbols from link step Remove npymath lib from _multiarray_umath since it contains symbols defined in object files being linked --- numpy/core/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index a4a84397d..16a34dd1e 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -941,7 +941,7 @@ def configuration(parent_package='',top_path=None): ], depends=deps + multiarray_deps + umath_deps + common_deps, - libraries=['npymath', 'npysort'], + libraries=['npysort'], extra_info=extra_info) ####################################################################### -- cgit v1.2.1 From 0d4faf4ec45e7835056107027191d29de7db549b Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Sun, 30 Aug 2020 12:54:22 +0100 Subject: BLD: Replace source files with lib Use lib to find symbols rather than object files --- numpy/core/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 16a34dd1e..71b4a7c0a 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -930,7 +930,7 @@ def configuration(parent_package='',top_path=None): config.add_extension('_multiarray_umath', sources=multiarray_src + umath_src + - npymath_sources + common_src + + common_src + [generate_config_h, generate_numpyconfig_h, generate_numpy_api, @@ -941,7 +941,7 @@ def configuration(parent_package='',top_path=None): ], depends=deps + multiarray_deps + umath_deps + common_deps, - libraries=['npysort'], + libraries=['npymath', 'npysort'], extra_info=extra_info) ####################################################################### -- cgit v1.2.1 From 6dbf65d00af42dea08b7b3c09952fb267cc1dac4 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2020 02:12:51 +0000 Subject: MAINT: Bump hypothesis from 5.26.0 to 5.30.0 Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 5.26.0 to 5.30.0. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-5.26.0...hypothesis-python-5.30.0) Signed-off-by: dependabot-preview[bot] --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 95c06e576..8aec9e7b1 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ cython==0.29.21 wheel setuptools<49.2.0 -hypothesis==5.26.0 +hypothesis==5.30.0 pytest==6.0.1 pytz==2020.1 pytest-cov==2.10.1 -- cgit v1.2.1 From 6e00aaedc40292933c0e8f162d0d4512d6ba350e Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 10:13:58 +0100 Subject: MAINT: Remove users of `numpy.compat.integer_types` Some more Python 2 cleanup --- numpy/fft/helper.py | 3 +-- numpy/fft/tests/test_helper.py | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/numpy/fft/helper.py b/numpy/fft/helper.py index 3dacd9ee1..927ee1af1 100644 --- a/numpy/fft/helper.py +++ b/numpy/fft/helper.py @@ -2,7 +2,6 @@ Discrete Fourier Transforms - helper.py """ -from numpy.compat import integer_types from numpy.core import integer, empty, arange, asarray, roll from numpy.core.overrides import array_function_dispatch, set_module @@ -10,7 +9,7 @@ from numpy.core.overrides import array_function_dispatch, set_module __all__ = ['fftshift', 'ifftshift', 'fftfreq', 'rfftfreq'] -integer_types = integer_types + (integer,) +integer_types = (int, integer) def _fftshift_dispatcher(x, axes=None): diff --git a/numpy/fft/tests/test_helper.py b/numpy/fft/tests/test_helper.py index 68f5990af..3fb700bb3 100644 --- a/numpy/fft/tests/test_helper.py +++ b/numpy/fft/tests/test_helper.py @@ -85,7 +85,6 @@ class TestFFTShift: def test_equal_to_original(self): """ Test that the new (>=v1.15) implementation (see #10073) is equal to the original (<=v1.14) """ - from numpy.compat import integer_types from numpy.core import asarray, concatenate, arange, take def original_fftshift(x, axes=None): @@ -94,7 +93,7 @@ class TestFFTShift: ndim = tmp.ndim if axes is None: axes = list(range(ndim)) - elif isinstance(axes, integer_types): + elif isinstance(axes, int): axes = (axes,) y = tmp for k in axes: @@ -110,7 +109,7 @@ class TestFFTShift: ndim = tmp.ndim if axes is None: axes = list(range(ndim)) - elif isinstance(axes, integer_types): + elif isinstance(axes, int): axes = (axes,) y = tmp for k in axes: -- cgit v1.2.1 From d5c841a7e55311e77a539f43f3dab58a68a1c9c1 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 10:17:39 +0100 Subject: MAINT: Remove users of `numpy.compat.strchar` Some more Python 2 cleanup --- numpy/core/tests/test_multiarray.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index a701de7c1..57650a33d 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -21,7 +21,6 @@ import builtins from decimal import Decimal import numpy as np -from numpy.compat import strchar import numpy.core._multiarray_tests as _multiarray_tests from numpy.testing import ( assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal, @@ -2031,7 +2030,7 @@ class TestMethods: strtype = '>i2' else: strtype = ' Date: Mon, 31 Aug 2020 10:21:25 +0100 Subject: MAINT: Remove users of `numpy.compat.open_latin1` Some more Python 2 cleanup. Also switches to use with statements, as previous passes looking for bare `open()` calls would have missed these --- numpy/distutils/fcompiler/__init__.py | 59 ++++++++++++++++------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index 1c3069363..a1c52412d 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -20,8 +20,6 @@ import os import sys import re -from numpy.compat import open_latin1 - from distutils.sysconfig import get_python_lib from distutils.fancy_getopt import FancyGetopt from distutils.errors import DistutilsModuleError, \ @@ -975,29 +973,27 @@ def is_free_format(file): # f90 allows both fixed and free format, assuming fixed unless # signs of free format are detected. result = 0 - f = open_latin1(file, 'r') - line = f.readline() - n = 10000 # the number of non-comment lines to scan for hints - if _has_f_header(line): - n = 0 - elif _has_f90_header(line): - n = 0 - result = 1 - while n>0 and line: - line = line.rstrip() - if line and line[0]!='!': - n -= 1 - if (line[0]!='\t' and _free_f90_start(line[:5])) or line[-1:]=='&': - result = 1 - break + with open(file, encoding='latin1') as f: line = f.readline() - f.close() + n = 10000 # the number of non-comment lines to scan for hints + if _has_f_header(line): + n = 0 + elif _has_f90_header(line): + n = 0 + result = 1 + while n>0 and line: + line = line.rstrip() + if line and line[0]!='!': + n -= 1 + if (line[0]!='\t' and _free_f90_start(line[:5])) or line[-1:]=='&': + result = 1 + break + line = f.readline() return result def has_f90_header(src): - f = open_latin1(src, 'r') - line = f.readline() - f.close() + with open(src, encoding='latin1') as f: + line = f.readline() return _has_f90_header(line) or _has_fix_header(line) _f77flags_re = re.compile(r'(c|)f77flags\s*\(\s*(?P\w+)\s*\)\s*=\s*(?P.*)', re.I) @@ -1008,17 +1004,16 @@ def get_f77flags(src): Return a dictionary {:}. """ flags = {} - f = open_latin1(src, 'r') - i = 0 - for line in f: - i += 1 - if i>20: break - m = _f77flags_re.match(line) - if not m: continue - fcname = m.group('fcname').strip() - fflags = m.group('fflags').strip() - flags[fcname] = split_quoted(fflags) - f.close() + with open(src, encoding='latin1') as f: + i = 0 + for line in f: + i += 1 + if i>20: break + m = _f77flags_re.match(line) + if not m: continue + fcname = m.group('fcname').strip() + fflags = m.group('fflags').strip() + flags[fcname] = split_quoted(fflags) return flags # TODO: implement get_f90flags and use it in _compile similarly to get_f77flags -- cgit v1.2.1 From 8a4fafa5278c2d894cf4efd40a342b33e941e373 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 10:23:54 +0100 Subject: MAINT: Remove users of `numpy.compat.bytes` Some more Python 2 cleanup. --- numpy/lib/_iotools.py | 2 +- numpy/lib/npyio.py | 2 +- numpy/lib/tests/test_io.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 7560bf4da..f5368526d 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -5,7 +5,7 @@ __docformat__ = "restructuredtext en" import numpy as np import numpy.core.numeric as nx -from numpy.compat import asbytes, asunicode, bytes +from numpy.compat import asbytes, asunicode def _decode_line(line, encoding=None): diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index cc3465cc6..d1a29ffad 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -23,7 +23,7 @@ from ._iotools import ( ) from numpy.compat import ( - asbytes, asstr, asunicode, bytes, os_fspath, os_PathLike, + asbytes, asstr, asunicode, os_fspath, os_PathLike, pickle, contextlib_nullcontext ) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index a23c6b007..38d698df4 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -19,7 +19,7 @@ from ctypes import c_bool import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConversionWarning -from numpy.compat import asbytes, bytes +from numpy.compat import asbytes from numpy.ma.testutils import assert_equal from numpy.testing import ( assert_warns, assert_, assert_raises_regex, assert_raises, -- cgit v1.2.1 From 36ad0e43127240377692b9ca669db06c7fb153f7 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 11:14:40 +0100 Subject: ENH: Make the window functions exactly symmetric This relies on the fact that `cos` is exactly symmetric around zero, but not around the floating-point approximation of `pi`. Closes gh-17169. --- numpy/lib/function_base.py | 16 ++++++++-------- numpy/lib/tests/test_function_base.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 710091de2..f1ec38c5c 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -2733,8 +2733,8 @@ def blackman(M): return array([]) if M == 1: return ones(1, float) - n = arange(0, M) - return 0.42 - 0.5*cos(2.0*pi*n/(M-1)) + 0.08*cos(4.0*pi*n/(M-1)) + n = arange(1-M, M, 2) + return 0.42 + 0.5*cos(pi*n/(M-1)) + 0.08*cos(2.0*pi*n/(M-1)) @set_module('numpy') @@ -2842,8 +2842,8 @@ def bartlett(M): return array([]) if M == 1: return ones(1, float) - n = arange(0, M) - return where(less_equal(n, (M-1)/2.0), 2.0*n/(M-1), 2.0 - 2.0*n/(M-1)) + n = arange(1-M, M, 2) + return where(less_equal(n, 0), 1 + n/(M-1), 1 - n/(M-1)) @set_module('numpy') @@ -2946,8 +2946,8 @@ def hanning(M): return array([]) if M == 1: return ones(1, float) - n = arange(0, M) - return 0.5 - 0.5*cos(2.0*pi*n/(M-1)) + n = arange(1-M, M, 2) + return 0.5 + 0.5*cos(pi*n/(M-1)) @set_module('numpy') @@ -3046,8 +3046,8 @@ def hamming(M): return array([]) if M == 1: return ones(1, float) - n = arange(0, M) - return 0.54 - 0.46*cos(2.0*pi*n/(M-1)) + n = arange(1-M, M, 2) + return 0.54 + 0.46*cos(pi*n/(M-1)) ## Code from cephes for i0 diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 41afccacc..7bddb941c 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1805,28 +1805,28 @@ class TestFilterwindows: def test_hanning(self): # check symmetry w = hanning(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) # check known value assert_almost_equal(np.sum(w, axis=0), 4.500, 4) def test_hamming(self): # check symmetry w = hamming(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) # check known value assert_almost_equal(np.sum(w, axis=0), 4.9400, 4) def test_bartlett(self): # check symmetry w = bartlett(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) # check known value assert_almost_equal(np.sum(w, axis=0), 4.4444, 4) def test_blackman(self): # check symmetry w = blackman(10) - assert_array_almost_equal(w, flipud(w), 7) + assert_equal(w, flipud(w)) # check known value assert_almost_equal(np.sum(w, axis=0), 3.7800, 4) -- cgit v1.2.1 From 6e935fed8019ac99f1f3d3a808a6ceae4bbbd421 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 12:43:14 +0100 Subject: MAINT: Improve error handling in npy_cpu_init It's generally unsafe to do a bunch of things then check `PyErr_Occurred()`, as once `PyErr_Occurred()` is non-null, many python api functions become unsafe. --- numpy/core/src/common/npy_cpu_features.c.src | 76 ++++++++++++++++------------ 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/numpy/core/src/common/npy_cpu_features.c.src b/numpy/core/src/common/npy_cpu_features.c.src index dfcf98c74..69bbc83a2 100644 --- a/numpy/core/src/common/npy_cpu_features.c.src +++ b/numpy/core/src/common/npy_cpu_features.c.src @@ -19,11 +19,11 @@ npy__cpu_init_features(void); * Multiple features can be present, and separated by space, comma, or tab. * Raises an error if parsing fails or if the feature was not enabled */ -static void +static int npy__cpu_try_disable_env(void); /* Ensure the build's CPU baseline features are supported at runtime */ -static void +static int npy__cpu_validate_baseline(void); /******************** Public Definitions *********************/ @@ -40,11 +40,12 @@ NPY_VISIBILITY_HIDDEN int npy_cpu_init(void) { npy__cpu_init_features(); - npy__cpu_validate_baseline(); - npy__cpu_try_disable_env(); - - if (PyErr_Occurred()) + if (npy__cpu_validate_baseline() < 0) { + return -1; + } + if (npy__cpu_try_disable_env() < 0) { return -1; + } return 0; } @@ -142,7 +143,7 @@ npy__cpu_dispatch_fid(const char *feature) return 0; } -static void +static int npy__cpu_validate_baseline(void) { #if !defined(NPY_DISABLE_OPTIMIZATION) && NPY_WITH_CPU_BASELINE_N > 0 @@ -165,16 +166,18 @@ npy__cpu_validate_baseline(void) "(" NPY_WITH_CPU_BASELINE ") but your machine doesn't support:\n(%s).", baseline_failure ); + return -1; } #endif + return 0; } -static void +static int npy__cpu_try_disable_env(void) { char *disenv = getenv("NPY_DISABLE_CPU_FEATURES"); if (disenv == NULL || disenv[0] == 0) { - return; + return 0; } #define NPY__CPU_ENV_ERR_HEAD \ "During parsing environment variable 'NPY_DISABLE_CPU_FEATURES':\n" @@ -187,7 +190,7 @@ npy__cpu_try_disable_env(void) "Length of environment variable 'NPY_DISABLE_CPU_FEATURES' is %d, only %d accepted", var_len, NPY__MAX_VAR_LEN - 1 ); - return; + return -1; } char disable_features[NPY__MAX_VAR_LEN]; memcpy(disable_features, disenv, var_len); @@ -210,7 +213,7 @@ npy__cpu_try_disable_env(void) "(" NPY_WITH_CPU_BASELINE ").", feature ); - break; + return -1; } // check if the feature is part of dispatched features int feature_id = npy__cpu_dispatch_fid(feature); @@ -236,36 +239,43 @@ npy__cpu_try_disable_env(void) *nexist_cur = '\0'; if (nexist[0] != '\0') { *(nexist_cur-1) = '\0'; // trim the last space - PyErr_WarnFormat(PyExc_RuntimeWarning, 1, - NPY__CPU_ENV_ERR_HEAD - "You cannot disable CPU features (%s), since " - "they are not part of the dispatched optimizations\n" - "(" NPY_WITH_CPU_DISPATCH ").", - nexist - ); + if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, + NPY__CPU_ENV_ERR_HEAD + "You cannot disable CPU features (%s), since " + "they are not part of the dispatched optimizations\n" + "(" NPY_WITH_CPU_DISPATCH ").", + nexist + ) < 0) { + return -1; + } } *notsupp_cur = '\0'; if (notsupp[0] != '\0') { *(notsupp_cur-1) = '\0'; // trim the last space - PyErr_WarnFormat(PyExc_RuntimeWarning, 1, - NPY__CPU_ENV_ERR_HEAD - "You cannot disable CPU features (%s), since " - "they are not supported by your machine.", - notsupp - ); + if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, + NPY__CPU_ENV_ERR_HEAD + "You cannot disable CPU features (%s), since " + "they are not supported by your machine.", + notsupp + ) < 0) { + return -1; + } } #else - PyErr_WarnFormat(PyExc_RuntimeWarning, 1, - NPY__CPU_ENV_ERR_HEAD - "You cannot use environment variable 'NPY_DISABLE_CPU_FEATURES', since " - #ifdef NPY_DISABLE_OPTIMIZATION - "the NumPy library was compiled with optimization disabled." - #else - "the NumPy library was compiled without any dispatched optimizations." - #endif - ); + if (PyErr_WarnFormat(PyExc_RuntimeWarning, 1, + NPY__CPU_ENV_ERR_HEAD + "You cannot use environment variable 'NPY_DISABLE_CPU_FEATURES', since " + #ifdef NPY_DISABLE_OPTIMIZATION + "the NumPy library was compiled with optimization disabled." + #else + "the NumPy library was compiled without any dispatched optimizations." + #endif + ) < 0) { + return -1; + } #endif + return 0; } /**************************************************************** -- cgit v1.2.1 From 6afc44d620272a032208c774aa766a11b2ea271e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 31 Aug 2020 10:11:09 -0500 Subject: DOC: Adopt (slightly modified) Ben's suggestions Making one of the in-text suggestions slightly adapted into a warning. --- numpy/core/_add_newdocs.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index 45c03f70d..a94b3193e 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -5496,9 +5496,15 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('metadata', Either ``None`` or a readonly dictionary of metadata (mappingproxy). The metadata field can be set using any dictionary at data-type - creation. Note that whether operations on arrays with metadata - attached to their datatypes propagate that metadata is currently - not well defined and should not be relied on. + creation. NumPy currently has no uniform approach to propagating + metadata; although some array operations preserve it there is no + guarantee that others will. + + .. warning:: + + Although used in certain projects this feature was long undocumented + and is not well supported. Some aspects of metadata propagation + are expected to change in the future. Examples -------- @@ -5510,13 +5516,13 @@ add_newdoc('numpy.core.multiarray', 'dtype', ('metadata', >>> arr.dtype.metadata mappingproxy({'key': 'value'}) - Some operations may preserve metadata (identical data types): + Adding arrays with identical datatypes currently preserves the metadata: >>> (arr + arr).dtype.metadata mappingproxy({'key': 'value'}) - But for example, adding two arrays with different metadata does not - propagate either one: + But if the arrays have different dtype metadata, the metadata may be + dropped: >>> dt2 = np.dtype(float, metadata={"key2": "value2"}) >>> arr2 = np.array([3, 2, 1], dtype=dt2) -- cgit v1.2.1 From 5dbc66ebe1fe524bec639316646ccc9dc18d8173 Mon Sep 17 00:00:00 2001 From: Eric Wieser Date: Mon, 31 Aug 2020 16:36:35 +0100 Subject: MAINT: Make the `NPY_CPU_DISPATCH_CALL` macros expressions not statements This means they can return values, and need semicolons --- numpy/core/code_generators/generate_umath.py | 2 +- numpy/core/src/common/npy_cpu_dispatch.h | 33 ++++++++++++++++------------ numpy/core/src/umath/_umath_tests.c.src | 10 ++++----- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 86e28b104..8ef9392d3 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -1042,7 +1042,7 @@ def make_arrays(funcdict): #ifndef NPY_DISABLE_OPTIMIZATION #include "{dname}.dispatch.h" #endif - NPY_CPU_DISPATCH_CALL_XB({name}_functions[{k}] = {tname}_{name}) + NPY_CPU_DISPATCH_CALL_XB({name}_functions[{k}] = {tname}_{name}); """).format( dname=dname, name=name, tname=tname, k=k )) diff --git a/numpy/core/src/common/npy_cpu_dispatch.h b/numpy/core/src/common/npy_cpu_dispatch.h index 846d1ebb9..274520852 100644 --- a/numpy/core/src/common/npy_cpu_dispatch.h +++ b/numpy/core/src/common/npy_cpu_dispatch.h @@ -217,44 +217,49 @@ * func_type the_callee(const int *src, int *dst, func_type *cb) * { * // direct call - * NPY_CPU_DISPATCH_CALL(dispatch_me, (src, dst)) + * NPY_CPU_DISPATCH_CALL(dispatch_me, (src, dst)); * // assign the pointer - * NPY_CPU_DISPATCH_CALL(*cb = dispatch_me) + * *cb = NPY_CPU_DISPATCH_CALL(dispatch_me); + * // or + * NPY_CPU_DISPATCH_CALL(*cb = dispatch_me); * // return the pointer - * NPY_CPU_DISPATCH_CALL(return dispatch_me) + * return NPY_CPU_DISPATCH_CALL(dispatch_me); * } */ #define NPY_CPU_DISPATCH_CALL(...) \ - if (0) {/*DUMMY*/} \ NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, NPY_CPU_DISPATCH_CALL_CB_, __VA_ARGS__) \ NPY__CPU_DISPATCH_BASELINE_CALL(NPY_CPU_DISPATCH_CALL_BASE_CB_, __VA_ARGS__) // Preprocessor callbacks #define NPY_CPU_DISPATCH_CALL_CB_(TESTED_FEATURES, TARGET_NAME, LEFT, ...) \ - else if (TESTED_FEATURES) { NPY_CAT(NPY_CAT(LEFT, _), TARGET_NAME) __VA_ARGS__; } + (TESTED_FEATURES) ? (NPY_CAT(NPY_CAT(LEFT, _), TARGET_NAME) __VA_ARGS__) : #define NPY_CPU_DISPATCH_CALL_BASE_CB_(LEFT, ...) \ - else { LEFT __VA_ARGS__; } + (LEFT __VA_ARGS__) /** * Macro NPY_CPU_DISPATCH_CALL_XB(LEFT, ...) * - * Same as `NPY_CPU_DISPATCH_DECLARE` but exclude the baseline declration even - * if it was provided within the configration statments. + * Same as `NPY_CPU_DISPATCH_DECLARE` but exclude the baseline declaration even + * if it was provided within the configration statements. + * Returns void. */ +#define NPY_CPU_DISPATCH_CALL_XB_CB_(TESTED_FEATURES, TARGET_NAME, LEFT, ...) \ + (TESTED_FEATURES) ? (void) (NPY_CAT(NPY_CAT(LEFT, _), TARGET_NAME) __VA_ARGS__) : #define NPY_CPU_DISPATCH_CALL_XB(...) \ - if (0) {/*DUMMY*/} \ - NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, NPY_CPU_DISPATCH_CALL_CB_, __VA_ARGS__) + NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, NPY_CPU_DISPATCH_CALL_XB_CB_, __VA_ARGS__) \ + ((void) 0 /* discarded expression value */) /** * Macro NPY_CPU_DISPATCH_CALL_ALL(LEFT, ...) * * Same as `NPY_CPU_DISPATCH_CALL` but dispatching all the required optimizations for * the exported functions and variables instead of highest interested one. + * Returns void. */ #define NPY_CPU_DISPATCH_CALL_ALL(...) \ - NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, NPY_CPU_DISPATCH_CALL_ALL_CB_, __VA_ARGS__) \ - NPY__CPU_DISPATCH_BASELINE_CALL(NPY_CPU_DISPATCH_CALL_ALL_BASE_CB_, __VA_ARGS__) + (NPY__CPU_DISPATCH_CALL(NPY_CPU_HAVE, NPY_CPU_DISPATCH_CALL_ALL_CB_, __VA_ARGS__) \ + NPY__CPU_DISPATCH_BASELINE_CALL(NPY_CPU_DISPATCH_CALL_ALL_BASE_CB_, __VA_ARGS__)) // Preprocessor callbacks #define NPY_CPU_DISPATCH_CALL_ALL_CB_(TESTED_FEATURES, TARGET_NAME, LEFT, ...) \ - if (TESTED_FEATURES) { NPY_CAT(NPY_CAT(LEFT, _), TARGET_NAME) __VA_ARGS__; } + ((TESTED_FEATURES) ? (NPY_CAT(NPY_CAT(LEFT, _), TARGET_NAME) __VA_ARGS__) : (void) 0), #define NPY_CPU_DISPATCH_CALL_ALL_BASE_CB_(LEFT, ...) \ - { LEFT __VA_ARGS__; } + ( LEFT __VA_ARGS__ ) #endif // NPY_CPU_DISPATCH_H_ diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index 3ab89d6a5..660c296d6 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -588,11 +588,11 @@ static PyObject * UMath_Tests_test_dispatch(PyObject *NPY_UNUSED(dummy), PyObject *NPY_UNUSED(dummy2)) { const char *highest_func, *highest_var; - NPY_CPU_DISPATCH_CALL(highest_func = _umath_tests_dispatch_func, ()) - NPY_CPU_DISPATCH_CALL(highest_var = _umath_tests_dispatch_var) + NPY_CPU_DISPATCH_CALL(highest_func = _umath_tests_dispatch_func, ()); + NPY_CPU_DISPATCH_CALL(highest_var = _umath_tests_dispatch_var); const char *highest_func_xb = "nobase", *highest_var_xb = "nobase"; - NPY_CPU_DISPATCH_CALL_XB(highest_func_xb = _umath_tests_dispatch_func, ()) - NPY_CPU_DISPATCH_CALL_XB(highest_var_xb = _umath_tests_dispatch_var) + NPY_CPU_DISPATCH_CALL_XB(highest_func_xb = _umath_tests_dispatch_func, ()); + NPY_CPU_DISPATCH_CALL_XB(highest_var_xb = _umath_tests_dispatch_var); PyObject *dict = PyDict_New(), *item; if (dict == NULL) { @@ -610,7 +610,7 @@ UMath_Tests_test_dispatch(PyObject *NPY_UNUSED(dummy), PyObject *NPY_UNUSED(dumm if (item == NULL || PyDict_SetItemString(dict, "all", item) < 0) { goto err; } - NPY_CPU_DISPATCH_CALL_ALL(_umath_tests_dispatch_attach, (item)) + NPY_CPU_DISPATCH_CALL_ALL(_umath_tests_dispatch_attach, (item)); if (PyErr_Occurred()) { goto err; } -- cgit v1.2.1 From 8d5b76789ea500ac7a032052c48cf737a76a5845 Mon Sep 17 00:00:00 2001 From: MelissaWM Date: Mon, 31 Aug 2020 17:03:05 -0300 Subject: DOC: Fixed headings for tutorials so they appear at new theme sidebar. --- doc/source/user/tutorial-ma.rst | 30 ++++++++++++++++++++---------- doc/source/user/tutorial-svd.rst | 33 ++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/doc/source/user/tutorial-ma.rst b/doc/source/user/tutorial-ma.rst index c28353371..88bad3cbe 100644 --- a/doc/source/user/tutorial-ma.rst +++ b/doc/source/user/tutorial-ma.rst @@ -9,7 +9,8 @@ Tutorial: Masked Arrays import numpy as np np.random.seed(1) -**Prerequisites** +Prerequisites +------------- Before reading this tutorial, you should know a bit of Python. If you would like to refresh your memory, take a look at the @@ -18,13 +19,15 @@ would like to refresh your memory, take a look at the If you want to be able to run the examples in this tutorial, you should also have `matplotlib `_ installed on your computer. -**Learner profile** +Learner profile +--------------- This tutorial is for people who have a basic understanding of NumPy and want to understand how masked arrays and the :mod:`numpy.ma` module can be used in practice. -**Learning Objectives** +Learning Objectives +------------------- After this tutorial, you should be able to: @@ -33,7 +36,8 @@ After this tutorial, you should be able to: - Decide when the use of masked arrays is appropriate in some of your applications -**What are masked arrays?** +What are masked arrays? +----------------------- Consider the following problem. You have a dataset with missing or invalid entries. If you're doing any kind of processing on this data, and want to @@ -63,7 +67,8 @@ combination of: - A ``fill_value``, a value that may be used to replace the invalid entries in order to return a standard :class:`numpy.ndarray`. -**When can they be useful?** +When can they be useful? +------------------------ There are a few situations where masked arrays can be more useful than just eliminating the invalid entries of an array: @@ -84,7 +89,8 @@ comes with a specific implementation of most :term:`NumPy universal functions functions and operations on masked data. The output is then a masked array. We'll see some examples of how this works in practice below. -**Using masked arrays to see COVID-19 data** +Using masked arrays to see COVID-19 data +---------------------------------------- From `Kaggle `_ it is possible to download a dataset with initial data about the COVID-19 outbreak in the @@ -149,7 +155,8 @@ can read more about the :func:`numpy.genfromtxt` function from the :func:`Reference Documentation ` or from the :doc:`Basic IO tutorial `. -**Exploring the data** +Exploring the data +------------------ First of all, we can plot the whole set of data we have and see what it looks like. In order to get a readable plot, we select only a few of the dates to @@ -194,7 +201,8 @@ the :func:`numpy.sum` function to sum all the selected rows (``axis=0``): Something's wrong with this data - we are not supposed to have negative values in a cumulative data set. What's going on? -**Missing data** +Missing data +------------ Looking at the data, here's what we find: there is a period with **missing data**: @@ -308,7 +316,8 @@ Mainland China: It's clear that masked arrays are the right solution here. We cannot represent the missing data without mischaracterizing the evolution of the curve. -**Fitting Data** +Fitting Data +------------ One possibility we can think of is to interpolate the missing data to estimate the number of cases in late January. Observe that we can select the masked @@ -367,7 +376,8 @@ after the beginning of the records: plt.title("COVID-19 cumulative cases from Jan 21 to Feb 3 2020 - Mainland China\n" "Cubic estimate for 7 days after start"); -**More reading** +More reading +------------ Topics not covered in this tutorial can be found in the documentation: diff --git a/doc/source/user/tutorial-svd.rst b/doc/source/user/tutorial-svd.rst index 086e0a6de..fd9e366e0 100644 --- a/doc/source/user/tutorial-svd.rst +++ b/doc/source/user/tutorial-svd.rst @@ -9,7 +9,8 @@ Tutorial: Linear algebra on n-dimensional arrays import numpy as np np.random.seed(1) -**Prerequisites** +Prerequisites +------------- Before reading this tutorial, you should know a bit of Python. If you would like to refresh your memory, take a look at the @@ -19,7 +20,8 @@ If you want to be able to run the examples in this tutorial, you should also have `matplotlib `_ and `SciPy `_ installed on your computer. -**Learner profile** +Learner profile +--------------- This tutorial is for people who have a basic understanding of linear algebra and arrays in NumPy and want to understand how n-dimensional @@ -28,7 +30,8 @@ you don't know how to apply common functions to n-dimensional arrays (without using for-loops), or if you want to understand axis and shape properties for n-dimensional arrays, this tutorial might be of help. -**Learning Objectives** +Learning Objectives +------------------- After this tutorial, you should be able to: @@ -38,7 +41,8 @@ After this tutorial, you should be able to: arrays without using for-loops; - Understand axis and shape properties for n-dimensional arrays. -**Content** +Content +------- In this tutorial, we will use a `matrix decomposition `_ from linear algebra, the @@ -78,7 +82,8 @@ We can see the image using the `matplotlib.pyplot.imshow` function:: If you are executing the commands above in the IPython shell, it might be necessary to use the command ``plt.show()`` to show the image window. -**Shape, axis and array properties** +Shape, axis and array properties +-------------------------------- Note that, in linear algebra, the dimension of a vector refers to the number of entries in an array. In NumPy, it instead defines the number of axes. For @@ -162,7 +167,8 @@ syntax:: >>> green_array = img_array[:, :, 1] >>> blue_array = img_array[:, :, 2] -**Operations on an axis** +Operations on an axis +--------------------- It is possible to use methods from linear algebra to approximate an existing set of data. Here, we will use the `SVD (Singular Value Decomposition) @@ -290,7 +296,8 @@ diagonal and with the appropriate dimensions for multiplying: in our case, Now, we want to check if the reconstructed ``U @ Sigma @ Vt`` is close to the original ``img_gray`` matrix. -**Approximation** +Approximation +------------- The `linalg` module includes a ``norm`` function, which computes the norm of a vector or matrix represented in a NumPy array. For @@ -360,7 +367,8 @@ Now, you can go ahead and repeat this experiment with other values of `k`, and each of your experiments should give you a slightly better (or worse) image depending on the value you choose. -**Applying to all colors** +Applying to all colors +---------------------- Now we want to do the same kind of operation, but to all three colors. Our first instinct might be to repeat the same operation we did above to each color @@ -411,7 +419,8 @@ matrices into the approximation. Now, note that To build the final approximation matrix, we must understand how multiplication across different axes works. -**Products with n-dimensional arrays** +Products with n-dimensional arrays +---------------------------------- If you have worked before with only one- or two-dimensional arrays in NumPy, you might use `numpy.dot` and `numpy.matmul` (or the ``@`` operator) @@ -495,7 +504,8 @@ Even though the image is not as sharp, using a small number of ``k`` singular values (compared to the original set of 768 values), we can recover many of the distinguishing features from this image. -**Final words** +Final words +----------- Of course, this is not the best method to *approximate* an image. However, there is, in fact, a result in linear algebra that says that the @@ -504,7 +514,8 @@ terms of the norm of the difference. For more information, see *G. H. Golub and C. F. Van Loan, Matrix Computations, Baltimore, MD, Johns Hopkins University Press, 1985*. -**Further reading** +Further reading +--------------- - :doc:`Python tutorial ` - :ref:`reference` -- cgit v1.2.1 From 459991afcddfce6fbbae3fa83467fe607f2de7d0 Mon Sep 17 00:00:00 2001 From: Chunlin Date: Tue, 1 Sep 2020 17:44:34 +0800 Subject: MAINT: Remove redundant headers (#17163) Some headers were included twice by the same file, this removes the redundant includes --- numpy/core/src/common/array_assign.c | 1 - numpy/core/src/multiarray/_multiarray_tests.c.src | 2 -- numpy/core/src/multiarray/common.c | 1 - numpy/core/src/multiarray/conversion_utils.c | 1 - numpy/core/src/multiarray/convert.c | 3 --- numpy/core/src/multiarray/multiarraymodule.c | 1 - numpy/core/src/umath/reduction.c | 1 - 7 files changed, 10 deletions(-) diff --git a/numpy/core/src/common/array_assign.c b/numpy/core/src/common/array_assign.c index e365b49e4..67abcae24 100644 --- a/numpy/core/src/common/array_assign.c +++ b/numpy/core/src/common/array_assign.c @@ -14,7 +14,6 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION #define _MULTIARRAYMODULE #include - #include "npy_config.h" #include "npy_pycompat.h" diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 9c1fa0bad..fc4b2647a 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -9,8 +9,6 @@ #include "common.h" #include "mem_overlap.h" #include "npy_extint128.h" -#include "common.h" - #if defined(MS_WIN32) || defined(__CYGWIN__) #define EXPORT(x) __declspec(dllexport) x diff --git a/numpy/core/src/multiarray/common.c b/numpy/core/src/multiarray/common.c index 2256906eb..6af71f351 100644 --- a/numpy/core/src/multiarray/common.c +++ b/numpy/core/src/multiarray/common.c @@ -12,7 +12,6 @@ #include "abstractdtypes.h" #include "usertypes.h" -#include "common.h" #include "npy_buffer.h" #include "get_attr_string.h" diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index 6ac7a2088..dd18f71fd 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -6,7 +6,6 @@ #define _MULTIARRAYMODULE #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" -#include "numpy/arrayobject.h" #include "npy_config.h" #include "npy_pycompat.h" diff --git a/numpy/core/src/multiarray/convert.c b/numpy/core/src/multiarray/convert.c index b68b9322d..29a2bb0e8 100644 --- a/numpy/core/src/multiarray/convert.c +++ b/numpy/core/src/multiarray/convert.c @@ -8,9 +8,6 @@ #define _MULTIARRAYMODULE #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" - -#include "npy_config.h" - #include "npy_pycompat.h" #include "common.h" diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index bc367b78d..8d5cbf3fa 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -65,7 +65,6 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "templ_common.h" /* for npy_mul_with_overflow_intp */ #include "compiled_base.h" #include "mem_overlap.h" -#include "alloc.h" #include "typeinfo.h" #include "get_attr_string.h" diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index d0fb2f6ed..f1423d8b9 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -16,7 +16,6 @@ #include "npy_config.h" #include -#include "npy_config.h" #include "npy_pycompat.h" #include "ctors.h" -- cgit v1.2.1 From 5a25de880501d641c01100740f0ad42c374a9505 Mon Sep 17 00:00:00 2001 From: Susan Chang Date: Tue, 1 Sep 2020 07:40:32 -0400 Subject: Add canonical url to doc template --- doc/source/_templates/layout.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html index 1be152265..ecfa08a61 100644 --- a/doc/source/_templates/layout.html +++ b/doc/source/_templates/layout.html @@ -1,6 +1,11 @@ {% extends "!layout.html" %} {%- block extrahead %} + +{% block linktags %} + +{% endblock %} +