From 09a52ed47bb26498c97a579ce1147861df696d84 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Thu, 28 Mar 2013 17:13:53 -0600 Subject: 2to3: Apply `imports` fixer. The `imports` fixer deals with the standard packages that have been renamed, removed, or methods that have moved. cPickle -- removed, use pickle commands -- removed, getoutput, getstatusoutput moved to subprocess urlparse -- removed, urlparse moved to urllib.parse cStringIO -- removed, use StringIO or io.StringIO copy_reg -- renamed copyreg _winreg -- renamed winreg ConfigParser -- renamed configparser __builtin__ -- renamed builtins In the case of `cPickle`, it is imported as `pickle` when python < 3 and performance may be a consideration, but otherwise plain old `pickle` is used. Dealing with `StringIO` is a bit tricky. There is an `io.StringIO` function in the `io` module, available since Python 2.6, but it expects unicode whereas `StringIO.StringIO` expects ascii. The Python 3 equivalent is then `io.BytesIO`. What I have done here is used BytesIO for anything that is emulating a file for testing purposes. That is more explicit than using a redefined StringIO as was done before we dropped support for Python 2.4 and 2.5. Closes #3180. --- doc/cdoc/numpyfilter.py | 6 +- doc/numpybook/runcode.py | 6 +- doc/sphinxext/numpydoc/comment_eater.py | 2 +- doc/sphinxext/numpydoc/compiler_unparse.py | 2 +- doc/sphinxext/numpydoc/docscrape.py | 2 +- doc/sphinxext/numpydoc/plot_directive.py | 2 +- numpy/__init__.py | 9 +- numpy/core/__init__.py | 6 +- numpy/core/numeric.py | 35 ++- numpy/core/numerictypes.py | 6 +- numpy/core/records.py | 4 +- numpy/core/setup.py | 26 +- numpy/core/tests/test_print.py | 6 +- numpy/core/tests/test_regression.py | 17 +- numpy/distutils/cpuinfo.py | 26 +- numpy/distutils/mingw32ccompiler.py | 1 - numpy/distutils/misc_util.py | 8 +- numpy/distutils/npy_pkg_config.py | 7 +- numpy/distutils/system_info.py | 1 + numpy/distutils/tests/test_exec_command.py | 15 +- numpy/f2py/__init__.py | 2 +- numpy/f2py/doc/collectinput.py | 8 +- numpy/lib/_datasource.py | 12 +- numpy/lib/_iotools.py | 6 +- numpy/lib/format.py | 11 +- numpy/lib/npyio.py | 18 +- numpy/lib/tests/test__datasource.py | 13 +- numpy/lib/tests/test__iotools.py | 11 +- numpy/lib/tests/test_format.py | 48 ++- numpy/lib/tests/test_io.py | 461 +++++++++++++++-------------- numpy/lib/tests/test_regression.py | 7 +- numpy/lib/tests/test_utils.py | 6 +- numpy/lib/utils.py | 7 +- numpy/linalg/lapack_lite/clapack_scrub.py | 4 +- numpy/ma/core.py | 42 +-- numpy/ma/tests/test_core.py | 28 +- numpy/ma/tests/test_mrecords.py | 27 +- numpy/numarray/functions.py | 3 +- numpy/oldnumeric/compat.py | 25 +- numpy/oldnumeric/misc.py | 13 +- numpy/testing/utils.py | 8 +- setup.py | 6 +- tools/py3tool.py | 3 +- tools/win32build/build.py | 8 +- 44 files changed, 513 insertions(+), 451 deletions(-) diff --git a/doc/cdoc/numpyfilter.py b/doc/cdoc/numpyfilter.py index 60bee1b51..7137df2d2 100755 --- a/doc/cdoc/numpyfilter.py +++ b/doc/cdoc/numpyfilter.py @@ -13,7 +13,11 @@ import re import os import textwrap import optparse -import cPickle as pickle + +if sys.version_info[0] >= 3: + import pickle +else: + import cPickle as pickle. CACHE_FILE = 'build/rst-cache.pck' diff --git a/doc/numpybook/runcode.py b/doc/numpybook/runcode.py index 66c74cd74..858168e11 100644 --- a/doc/numpybook/runcode.py +++ b/doc/numpybook/runcode.py @@ -18,7 +18,7 @@ from __future__ import division, absolute_import import sys import optparse -import cStringIO +import io import re import os @@ -27,7 +27,7 @@ newre = re.compile(r"\\begin_inset Note.*PYNEW\s+\\end_inset", re.DOTALL) def getoutput(tstr, dic): print "\n\nRunning..." print tstr, - tempstr = cStringIO.StringIO() + tempstr = io.StringIO() sys.stdout = tempstr code = compile(tstr, '', 'exec') try: @@ -82,7 +82,7 @@ def getnewcodestr(substr, dic): def runpycode(lyxstr, name='MyCode'): schobj = re.compile(r"\\layout %s\s+>>> " % name) - outstr = cStringIO.StringIO() + outstr = io.StringIO() num = 0 indx = [] for it in schobj.finditer(lyxstr): diff --git a/doc/sphinxext/numpydoc/comment_eater.py b/doc/sphinxext/numpydoc/comment_eater.py index 74d0d4768..2c7f4da31 100644 --- a/doc/sphinxext/numpydoc/comment_eater.py +++ b/doc/sphinxext/numpydoc/comment_eater.py @@ -4,7 +4,7 @@ import sys if sys.version_info[0] >= 3: from io import StringIO else: - from cStringIO import StringIO + from io import StringIO import compiler import inspect diff --git a/doc/sphinxext/numpydoc/compiler_unparse.py b/doc/sphinxext/numpydoc/compiler_unparse.py index bb76f7ea3..c36389d13 100644 --- a/doc/sphinxext/numpydoc/compiler_unparse.py +++ b/doc/sphinxext/numpydoc/compiler_unparse.py @@ -18,7 +18,7 @@ from compiler.ast import Const, Name, Tuple, Div, Mul, Sub, Add if sys.version_info[0] >= 3: from io import StringIO else: - from cStringIO import StringIO + from io import StringIO def unparse(ast, single_line_functions=False): s = StringIO() diff --git a/doc/sphinxext/numpydoc/docscrape.py b/doc/sphinxext/numpydoc/docscrape.py index af76b86e5..59d3f9734 100644 --- a/doc/sphinxext/numpydoc/docscrape.py +++ b/doc/sphinxext/numpydoc/docscrape.py @@ -14,7 +14,7 @@ import collections if sys.version_info[0] >= 3: from io import StringIO else: - from cStringIO import StringIO + from io import StringIO class Reader(object): """A line-based string reader. diff --git a/doc/sphinxext/numpydoc/plot_directive.py b/doc/sphinxext/numpydoc/plot_directive.py index 9e92acabb..a7f4655ce 100644 --- a/doc/sphinxext/numpydoc/plot_directive.py +++ b/doc/sphinxext/numpydoc/plot_directive.py @@ -82,7 +82,7 @@ import sphinx if sys.version_info[0] >= 3: from io import StringIO else: - from cStringIO import StringIO + from io import StringIO import warnings warnings.warn("A plot_directive module is also available under " diff --git a/numpy/__init__.py b/numpy/__init__.py index 9cae9d388..f8c1de5ce 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -106,6 +106,8 @@ Exceptions to this rule are documented. """ from __future__ import division, absolute_import +import sys + # We first need to detect if we're being called as part of the numpy setup # procedure itself in a reliable manner. try: @@ -160,8 +162,11 @@ else: # Make these accessible from numpy name-space # but not imported in from numpy import * - from __builtin__ import bool, int, long, float, complex, \ - object, unicode, str + if sys.version_info[0] >= 3: + from builtins import bool, int, long, float, complex, object, unicode, str + else: + from __builtin__ import bool, int, long, float, complex, object, unicode, str + from .core import round, abs, max, min __all__.extend(['__version__', 'pkgload', 'PackageLoader', diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index f055d289e..d2f7c3c8c 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -62,10 +62,10 @@ def _ufunc_reduce(func): import sys -if sys.version_info[0] < 3: - import copy_reg as copyreg -else: +if sys.version_info[0] >= 3: import copyreg +else: + import copy_reg as copyreg copyreg.pickle(ufunc, _ufunc_reduce, _ufunc_reconstruct) # Unclutter namespace (must keep _ufunc_reconstruct for unpickling) diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 25f977254..5f4504eb9 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -1,5 +1,22 @@ from __future__ import division, absolute_import +import sys +import warnings +from . import multiarray +from . import umath +from .umath import * +from . import numerictypes +from .numerictypes import * +import collections + +if sys.version_info[0] >= 3: + import pickle +else: + import cPickle as pickle + +loads = pickle.loads + + __all__ = ['newaxis', 'ndarray', 'flatiter', 'nditer', 'nested_iters', 'ufunc', 'arange', 'array', 'zeros', 'count_nonzero', 'empty', 'broadcast', 'dtype', 'fromstring', 'fromfile', @@ -24,19 +41,10 @@ __all__ = ['newaxis', 'ndarray', 'flatiter', 'nditer', 'nested_iters', 'ufunc', 'CLIP', 'RAISE', 'WRAP', 'MAXDIMS', 'BUFSIZE', 'ALLOW_THREADS', 'ComplexWarning'] -import sys -import warnings -from . import multiarray -from . import umath -from .umath import * -from . import numerictypes -from .numerictypes import * -import collections - - if sys.version_info[0] < 3: __all__.extend(['getbuffer', 'newbuffer']) + class ComplexWarning(RuntimeWarning): """ The warning raised when casting a complex dtype to a real dtype. @@ -1861,9 +1869,6 @@ def base_repr(number, base=2, padding=0): res.append('-') return ''.join(reversed(res or '0')) -from cPickle import load, loads -_cload = load -_file = open def load(file): """ @@ -1880,8 +1885,8 @@ def load(file): """ if isinstance(file, type("")): - file = _file(file,"rb") - return _cload(file) + file = open(file, "rb") + return pickle.load(file) # These are all essentially abbreviations # These might wind up in a special abbreviations module diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index a1af9d80d..8bf0cc880 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -98,7 +98,11 @@ import sys # we don't export these for import *, but we do want them accessible # as numerictypes.bool, etc. -from __builtin__ import bool, int, long, float, complex, object, unicode, str +if sys.version_info[0] >= 3: + from builtins import bool, int, long, float, complex, object, unicode, str +else: + from __builtin__ import bool, int, long, float, complex, object, unicode, str + from numpy.compat import bytes if sys.version_info[0] >= 3: diff --git a/numpy/core/records.py b/numpy/core/records.py index 385f9866d..7a9481b38 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -595,8 +595,8 @@ def fromrecords(recList, dtype=None, shape=None, formats=None, names=None, >>> r.col2 chararray(['dbe', 'de'], dtype='|S3') - >>> import cPickle - >>> print cPickle.loads(cPickle.dumps(r)) + >>> import pickle + >>> print pickle.loads(pickle.dumps(r)) [(456, 'dbe', 1.2) (2, 'de', 1.3)] """ diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 37f649e6b..9fe141daf 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -4,12 +4,14 @@ import imp import os import sys import shutil +import pickle +import copy +import warnings +import re from os.path import join from numpy.distutils import log from distutils.dep_util import newer from distutils.sysconfig import get_config_var -import warnings -import re from setup_common import * @@ -22,11 +24,9 @@ ENABLE_SEPARATE_COMPILATION = (os.environ.get('NPY_SEPARATE_COMPILATION', "1") ! # configuration informations between extensions is not easy. # Using a pickled-based memoize does not work because config_cmd is an instance # method, which cPickle does not like. -try: - import cPickle as _pik -except ImportError: - import pickle as _pik -import copy +# +# Use pickle in all cases, as cPickle is gone in python3 and the difference +# in time is only in build. -- Charles Harris, 2013-03-30 class CallOnceOnly(object): def __init__(self): @@ -37,25 +37,25 @@ class CallOnceOnly(object): def check_types(self, *a, **kw): if self._check_types is None: out = check_types(*a, **kw) - self._check_types = _pik.dumps(out) + self._check_types = pickle.dumps(out) else: - out = copy.deepcopy(_pik.loads(self._check_types)) + out = copy.deepcopy(pickle.loads(self._check_types)) return out def check_ieee_macros(self, *a, **kw): if self._check_ieee_macros is None: out = check_ieee_macros(*a, **kw) - self._check_ieee_macros = _pik.dumps(out) + self._check_ieee_macros = pickle.dumps(out) else: - out = copy.deepcopy(_pik.loads(self._check_ieee_macros)) + out = copy.deepcopy(pickle.loads(self._check_ieee_macros)) return out def check_complex(self, *a, **kw): if self._check_complex is None: out = check_complex(*a, **kw) - self._check_complex = _pik.dumps(out) + self._check_complex = pickle.dumps(out) else: - out = copy.deepcopy(_pik.loads(self._check_complex)) + out = copy.deepcopy(pickle.loads(self._check_complex)) return out PYTHON_HAS_UNICODE_WIDE = True diff --git a/numpy/core/tests/test_print.py b/numpy/core/tests/test_print.py index 67021c20e..e2469ec7b 100644 --- a/numpy/core/tests/test_print.py +++ b/numpy/core/tests/test_print.py @@ -6,7 +6,11 @@ import nose import locale import sys -from StringIO import StringIO + +if sys.version_info[0] >= 3: + from io import StringIO +else: + from StringIO import StringIO _REF = {np.inf: 'inf', -np.inf: '-inf', np.nan: 'nan'} diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 19c15b32f..575fb381c 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -7,8 +7,9 @@ import gc import copy import warnings import tempfile -from StringIO import StringIO from os import path +from io import BytesIO + import numpy as np from numpy.testing import ( run_module_suite, TestCase, assert_, assert_equal, @@ -18,10 +19,6 @@ from numpy.testing import ( from numpy.testing.utils import _assert_valid_refcount, WarningManager from numpy.compat import asbytes, asunicode, asbytes_nested -if sys.version_info[0] >= 3: - import io - StringIO = io.BytesIO - rlevel = 1 class TestRegression(TestCase): @@ -37,7 +34,7 @@ class TestRegression(TestCase): def test_pickle_transposed(self,level=rlevel): """Ticket #16""" a = np.transpose(np.array([[2,9],[7,0],[3,8]])) - f = StringIO() + f = BytesIO() pickle.dump(a,f) f.seek(0) b = pickle.load(f) @@ -90,7 +87,7 @@ class TestRegression(TestCase): def test_char_dump(self,level=rlevel): """Ticket #50""" - f = StringIO() + f = BytesIO() ca = np.char.array(np.arange(1000,1010),itemsize=4) ca.dump(f) f.seek(0) @@ -322,7 +319,7 @@ class TestRegression(TestCase): def test_unpickle_dtype_with_object(self,level=rlevel): """Implemented in r2840""" dt = np.dtype([('x',int),('y',np.object_),('z','O')]) - f = StringIO() + f = BytesIO() pickle.dump(dt,f) f.seek(0) dt_ = pickle.load(f) @@ -386,7 +383,6 @@ class TestRegression(TestCase): def test_pickle_dtype(self,level=rlevel): """Ticket #251""" - import pickle pickle.dumps(np.float) def test_swap_real(self, level=rlevel): @@ -722,10 +718,9 @@ class TestRegression(TestCase): def test_unicode_scalar(self, level=rlevel): """Ticket #600""" - import cPickle x = np.array(["DROND", "DROND1"], dtype="U6") el = x[1] - new = cPickle.loads(cPickle.dumps(el)) + new = pickle.loads(pickle.dumps(el)) assert_equal(new, el) def test_arange_non_native_dtype(self, level=rlevel): diff --git a/numpy/distutils/cpuinfo.py b/numpy/distutils/cpuinfo.py index 1e2d9379c..aaf642691 100644 --- a/numpy/distutils/cpuinfo.py +++ b/numpy/distutils/cpuinfo.py @@ -18,10 +18,12 @@ __all__ = ['cpu'] import sys, re, types import os -if sys.version_info[0] < 3: - from commands import getstatusoutput -else: + +if sys.version_info[0] >= 3: from subprocess import getstatusoutput +else: + from commands import getstatusoutput + import warnings import platform @@ -488,25 +490,29 @@ class Win32CPUInfo(CPUInfoBase): info = [] try: #XXX: Bad style to use so long `try:...except:...`. Fix it! - import _winreg + if sys.version_info[0] >= 3: + import winreg + else: + import _winreg as winreg + prgx = re.compile(r"family\s+(?P\d+)\s+model\s+(?P\d+)"\ "\s+stepping\s+(?P\d+)",re.IGNORECASE) - chnd=_winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, self.pkey) + chnd=winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, self.pkey) pnum=0 while 1: try: - proc=_winreg.EnumKey(chnd,pnum) - except _winreg.error: + proc=winreg.EnumKey(chnd,pnum) + except winreg.error: break else: pnum+=1 info.append({"Processor":proc}) - phnd=_winreg.OpenKey(chnd,proc) + phnd=winreg.OpenKey(chnd,proc) pidx=0 while True: try: - name,value,vtpe=_winreg.EnumValue(phnd,pidx) - except _winreg.error: + name,value,vtpe=winreg.EnumValue(phnd,pidx) + except winreg.error: break else: pidx=pidx+1 diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 0505309fa..ed0dc0d4e 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -10,7 +10,6 @@ Support code for building Python extensions on Windows. from __future__ import division, absolute_import import os -import subprocess import sys import subprocess import re diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index f526557d6..0c88e0ae4 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -2125,9 +2125,13 @@ def get_info(pkgname, dirs=None): return info def is_bootstrapping(): - import __builtin__ + if sys.version_info[0] >= 3: + import builtins + else: + import __builtin__ as builtins + try: - __builtin__.__NUMPY_SETUP__ + builtins.__NUMPY_SETUP__ return True except AttributeError: return False diff --git a/numpy/distutils/npy_pkg_config.py b/numpy/distutils/npy_pkg_config.py index 2dacbbc70..a0d75670b 100644 --- a/numpy/distutils/npy_pkg_config.py +++ b/numpy/distutils/npy_pkg_config.py @@ -1,13 +1,14 @@ from __future__ import division, absolute_import import sys +import re +import os +import shlex + if sys.version_info[0] < 3: from ConfigParser import SafeConfigParser, NoOptionError else: from configparser import ConfigParser, SafeConfigParser, NoOptionError -import re -import os -import shlex __all__ = ['FormatError', 'PkgNotFound', 'LibraryInfo', 'VariableSet', 'read_config', 'parse_flags'] diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 68ad926e3..ff32daf69 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -118,6 +118,7 @@ import re import copy import warnings from glob import glob + if sys.version_info[0] < 3: from ConfigParser import NoOptionError, ConfigParser else: diff --git a/numpy/distutils/tests/test_exec_command.py b/numpy/distutils/tests/test_exec_command.py index 11f262369..3e85fcd3e 100644 --- a/numpy/distutils/tests/test_exec_command.py +++ b/numpy/distutils/tests/test_exec_command.py @@ -2,11 +2,16 @@ from __future__ import division, absolute_import import os import sys -import StringIO from tempfile import TemporaryFile from numpy.distutils import exec_command +# In python 3 stdout, stderr are text (unicode compliant) devices, so to +# emulate them import StringIO from the io module. +if sys.version_info[0] >= 3: + from io import StringIO +else: + from StringIO import StringIO class redirect_stdout(object): """Context manager to redirect stdout for exec_command test.""" @@ -62,26 +67,26 @@ def test_exec_command_stdout(): # both that the special case works and that the generic code works. # Test posix version: - with redirect_stdout(StringIO.StringIO()): + with redirect_stdout(StringIO()): with redirect_stderr(TemporaryFile()): exec_command.exec_command("cd '.'") if os.name == 'posix': # Test general (non-posix) version: with emulate_nonposix(): - with redirect_stdout(StringIO.StringIO()): + with redirect_stdout(StringIO()): with redirect_stderr(TemporaryFile()): exec_command.exec_command("cd '.'") def test_exec_command_stderr(): # Test posix version: with redirect_stdout(TemporaryFile(mode='w+')): - with redirect_stderr(StringIO.StringIO()): + with redirect_stderr(StringIO()): exec_command.exec_command("cd '.'") if os.name == 'posix': # Test general (non-posix) version: with emulate_nonposix(): with redirect_stdout(TemporaryFile()): - with redirect_stderr(StringIO.StringIO()): + with redirect_stderr(StringIO()): exec_command.exec_command("cd '.'") diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index 951544588..6b20f94c4 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -5,7 +5,7 @@ __all__ = ['run_main','compile','f2py_testing'] import os import sys -import commands +import subprocess from . import f2py2e from . import f2py_testing diff --git a/numpy/f2py/doc/collectinput.py b/numpy/f2py/doc/collectinput.py index 078da8eeb..396e4a912 100755 --- a/numpy/f2py/doc/collectinput.py +++ b/numpy/f2py/doc/collectinput.py @@ -27,7 +27,11 @@ stdoutflag=0 import sys import fileinput import re -import commands + +if sys.version_info[0] >= 3: + from subprocess import getoutput +else: + from commands import getoutput try: fn=sys.argv[2] except: @@ -69,7 +73,7 @@ for l in fileinput.input(fi): elif flag==1: sys.stderr.write(fn+'\n') print '%%%%% Begin of '+fn - print commands.getoutput(sys.argv[0]+' < '+fn) + print getoutput(sys.argv[0]+' < '+fn) print '%%%%% End of '+fn else: sys.stderr.write('Could not extract a file name from: '+l) diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index db97c5507..9f0bb1ae8 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -36,6 +36,7 @@ from __future__ import division, absolute_import __docformat__ = "restructuredtext en" import os +import sys from shutil import rmtree, copyfile, copyfileobj _open = open @@ -252,7 +253,10 @@ class DataSource (object): """Test if path is a net location. Tests the scheme and netloc.""" # We do this here to reduce the 'import numpy' initial import time. - from urlparse import urlparse + if sys.version_info[0] >= 3: + from urllib.parse import urlparse + else: + from urlparse import urlparse # BUG : URLs require a scheme string ('http://') to be used. # www.google.com will fail. @@ -351,8 +355,10 @@ class DataSource (object): """ # We do this here to reduce the 'import numpy' initial import time. - from urlparse import urlparse - + if sys.version_info[0] >= 3: + from urllib.parse import urlparse + else: + from urlparse import urlparse # TODO: This should be more robust. Handles case where path includes # the destpath, but not other sub-paths. Failing case: diff --git a/numpy/lib/_iotools.py b/numpy/lib/_iotools.py index 87ce9b01d..b8a01bafc 100644 --- a/numpy/lib/_iotools.py +++ b/numpy/lib/_iotools.py @@ -8,7 +8,11 @@ __docformat__ = "restructuredtext en" import sys import numpy as np import numpy.core.numeric as nx -from __builtin__ import bool, int, long, float, complex, object, unicode, str + +if sys.version_info[0] >= 3: + from builtins import bool, int, long, float, complex, object, unicode, str +else: + from __builtin__ import bool, int, long, float, complex, object, unicode, str from numpy.compat import asbytes, bytes, asbytes_nested diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 21b0607e7..dedfabfd2 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -136,13 +136,16 @@ alternatives, is described fully in the "npy-format" NEP. """ from __future__ import division, absolute_import -import cPickle - import numpy import sys from numpy.lib.utils import safe_eval from numpy.compat import asbytes, isfileobj +if sys.version_info[0] >= 3: + import pickle +else: + import cPickle as pickle + MAGIC_PREFIX = asbytes('\x93NUMPY') MAGIC_LEN = len(MAGIC_PREFIX) + 2 @@ -399,7 +402,7 @@ def write_array(fp, array, version=(1,0)): if array.dtype.hasobject: # We contain Python objects so we cannot write out the data directly. # Instead, we will pickle it out with version 2 of the pickle protocol. - cPickle.dump(array, fp, protocol=2) + pickle.dump(array, fp, protocol=2) elif array.flags.f_contiguous and not array.flags.c_contiguous: if isfileobj(fp): array.T.tofile(fp) @@ -447,7 +450,7 @@ def read_array(fp): # Now read the actual data. if dtype.hasobject: # The array contained Python objects. We need to unpickle the data. - array = cPickle.load(fp) + array = pickle.load(fp) else: if isfileobj(fp): # We can use the fast fromfile() function. diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 065a3ba46..eb4ffd4ce 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -1,9 +1,5 @@ from __future__ import division, absolute_import -__all__ = ['savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt', - 'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez', - 'savez_compressed', 'packbits', 'unpackbits', 'fromregex', 'DataSource'] - import numpy as np from . import format import sys @@ -15,7 +11,6 @@ import warnings import weakref from operator import itemgetter -from cPickle import load as _cload, loads from ._datasource import DataSource from ._compiled_base import packbits, unpackbits @@ -25,11 +20,18 @@ from ._iotools import LineSplitter, NameValidator, StringConverter, \ easy_dtype, _bytes_to_name from numpy.compat import asbytes, asstr, asbytes_nested, bytes +from io import BytesIO if sys.version_info[0] >= 3: - from io import BytesIO + import pickle else: - from cStringIO import StringIO as BytesIO + import cPickle as pickle + +loads = pickle.loads + +__all__ = ['savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt', + 'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez', + 'savez_compressed', 'packbits', 'unpackbits', 'fromregex', 'DataSource'] _string_like = _is_string_like @@ -380,7 +382,7 @@ def load(file, mmap_mode=None): return format.read_array(fid) else: # Try a pickle try: - return _cload(fid) + return pickle.load(fid) except: raise IOError( "Failed to interpret file %s as a pickle" % repr(file)) diff --git a/numpy/lib/tests/test__datasource.py b/numpy/lib/tests/test__datasource.py index e784e4296..ccd7405fe 100644 --- a/numpy/lib/tests/test__datasource.py +++ b/numpy/lib/tests/test__datasource.py @@ -1,17 +1,20 @@ from __future__ import division, absolute_import import os +import urllib2 +import sys +import numpy.lib._datasource as datasource from tempfile import mkdtemp, mkstemp, NamedTemporaryFile from shutil import rmtree -from urlparse import urlparse from urllib2 import URLError -import urllib2 - +from numpy.compat import asbytes from numpy.testing import * -from numpy.compat import asbytes -import numpy.lib._datasource as datasource +if sys.version_info[0] >= 3: + from urllib.parse import urlparse +else: + from urlparse import urlparse def urlopen_stub(url, data=None): '''Stub to replace urlopen for testing.''' diff --git a/numpy/lib/tests/test__iotools.py b/numpy/lib/tests/test__iotools.py index ebb01ad3a..1cd9a25da 100644 --- a/numpy/lib/tests/test__iotools.py +++ b/numpy/lib/tests/test__iotools.py @@ -1,22 +1,13 @@ from __future__ import division, absolute_import import sys - -if sys.version_info[0] >= 3: - from io import BytesIO - def StringIO(s=""): - return BytesIO(asbytes(s)) -else: - from StringIO import StringIO - -from datetime import date import time +from datetime import date import numpy as np from numpy.lib._iotools import LineSplitter, NameValidator, StringConverter, \ has_nested_fields, easy_dtype, flatten_dtype from numpy.testing import * - from numpy.compat import asbytes, asbytes_nested class TestLineSplitter(TestCase): diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index 5c07da1c3..aae3b1ba8 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -5,10 +5,7 @@ r''' Test the .npy file format. Set up: >>> import sys - >>> if sys.version_info[0] >= 3: - ... from io import BytesIO as StringIO - ... else: - ... from cStringIO import StringIO + >>> from io import BytesIO >>> from numpy.lib import format >>> >>> scalars = [ @@ -101,19 +98,19 @@ Test the magic string writing. Test the magic string reading. - >>> format.read_magic(StringIO(format.magic(1, 0))) + >>> format.read_magic(BytesIO(format.magic(1, 0))) (1, 0) - >>> format.read_magic(StringIO(format.magic(0, 0))) + >>> format.read_magic(BytesIO(format.magic(0, 0))) (0, 0) - >>> format.read_magic(StringIO(format.magic(255, 255))) + >>> format.read_magic(BytesIO(format.magic(255, 255))) (255, 255) - >>> format.read_magic(StringIO(format.magic(2, 5))) + >>> format.read_magic(BytesIO(format.magic(2, 5))) (2, 5) Test the header writing. >>> for arr in basic_arrays + record_arrays: - ... f = StringIO() + ... f = BytesIO() ... format.write_array_header_1_0(f, arr) # XXX: arr is not a dict, items gets called on it ... print repr(f.getvalue()) ... @@ -279,22 +276,15 @@ Test the header writing. "\x16\x02{'descr': [('x', '>i4', (2,)),\n ('Info',\n [('value', '>c16'),\n ('y2', '>f8'),\n ('Info2',\n [('name', '|S2'),\n ('value', '>c16', (2,)),\n ('y3', '>f8', (2,)),\n ('z3', '>u4', (2,))]),\n ('name', '|S2'),\n ('z2', '|b1')]),\n ('color', '|S2'),\n ('info', [('Name', '>U8'), ('Value', '>c16')]),\n ('y', '>f8', (2, 2)),\n ('z', '|u1')],\n 'fortran_order': False,\n 'shape': (2,)} \n" ''' - import sys import os import shutil import tempfile - -if sys.version_info[0] >= 3: - from io import BytesIO as StringIO -else: - from cStringIO import StringIO +from io import BytesIO import numpy as np from numpy.testing import * - from numpy.lib import format - from numpy.compat import asbytes, asbytes_nested @@ -416,9 +406,9 @@ record_arrays = [ ] def roundtrip(arr): - f = StringIO() + f = BytesIO() format.write_array(f, arr) - f2 = StringIO(f.getvalue()) + f2 = BytesIO(f.getvalue()) arr2 = format.read_array(f2) return arr2 @@ -469,7 +459,7 @@ def test_memmap_roundtrip(): def test_write_version_1_0(): - f = StringIO() + f = BytesIO() arr = np.arange(1) # These should pass. format.write_array(f, arr, version=(1, 0)) @@ -513,12 +503,12 @@ malformed_magic = asbytes_nested([ def test_read_magic_bad_magic(): for magic in malformed_magic: - f = StringIO(magic) + f = BytesIO(magic) yield raises(ValueError)(format.read_magic), f def test_read_version_1_0_bad_magic(): for magic in bad_version_magic + malformed_magic: - f = StringIO(magic) + f = BytesIO(magic) yield raises(ValueError)(format.read_array), f def test_bad_magic_args(): @@ -528,29 +518,29 @@ def test_bad_magic_args(): assert_raises(ValueError, format.magic, 1, 256) def test_large_header(): - s = StringIO() + s = BytesIO() d = {'a':1,'b':2} format.write_array_header_1_0(s,d) - s = StringIO() + s = BytesIO() d = {'a':1,'b':2,'c':'x'*256*256} assert_raises(ValueError, format.write_array_header_1_0, s, d) def test_bad_header(): # header of length less than 2 should fail - s = StringIO() + s = BytesIO() assert_raises(ValueError, format.read_array_header_1_0, s) - s = StringIO(asbytes('1')) + s = BytesIO(asbytes('1')) assert_raises(ValueError, format.read_array_header_1_0, s) # header shorter than indicated size should fail - s = StringIO(asbytes('\x01\x00')) + s = BytesIO(asbytes('\x01\x00')) assert_raises(ValueError, format.read_array_header_1_0, s) # headers without the exact keys required should fail d = {"shape":(1,2), "descr":"x"} - s = StringIO() + s = BytesIO() format.write_array_header_1_0(s,d) assert_raises(ValueError, format.read_array_header_1_0, s) @@ -558,7 +548,7 @@ def test_bad_header(): "fortran_order":False, "descr":"x", "extrakey":-1} - s = StringIO() + s = BytesIO() format.write_array_header_1_0(s,d) assert_raises(ValueError, format.read_array_header_1_0, s) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index ab990976e..6b5173890 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -4,31 +4,43 @@ import sys import gzip import os import threading -from tempfile import mkstemp, NamedTemporaryFile import time -from datetime import datetime import warnings import gc +from tempfile import mkstemp, NamedTemporaryFile +from io import BytesIO +from datetime import datetime from numpy.testing.utils import WarningManager import numpy as np import numpy.ma as ma from numpy.lib._iotools import ConverterError, ConverterLockError, \ ConversionWarning -from numpy.compat import asbytes, asbytes_nested, bytes - +from numpy.compat import asbytes, asbytes_nested, bytes, asstr from nose import SkipTest from numpy.ma.testutils import (TestCase, assert_equal, assert_array_equal, assert_raises, run_module_suite) from numpy.testing import assert_warns, assert_, build_err_msg -if sys.version_info[0] >= 3: - from io import BytesIO - def StringIO(s=""): - return BytesIO(asbytes(s)) -else: - from StringIO import StringIO - BytesIO = StringIO + +class TextIO(BytesIO): + """Helper IO class. + + Writes encode strings to bytes if needed, reads return bytes. + This makes it easier to emulate files opened in binary mode + without needing to explicitly convert strings to bytes in + setting up the test data. + + """ + def __init__(self, s=""): + BytesIO.__init__(self, asbytes(s)) + + def write(self, s): + BytesIO.write(self, asbytes(s)) + + def writelines(self, lines): + BytesIO.writelines(self, [asbytes(s) for s in lines]) + MAJVER, MINVER = sys.version_info[:2] @@ -73,7 +85,7 @@ class RoundtripTest(object): target_file = NamedTemporaryFile() load_file = target_file.name else: - target_file = StringIO() + target_file = BytesIO() load_file = target_file arr = args @@ -135,7 +147,7 @@ class TestSavezLoad(RoundtripTest, TestCase): def test_named_arrays(self): a = np.array([[1, 2], [3, 4]], float) b = np.array([[1 + 2j, 2 + 7j], [3 - 6j, 4 + 12j]], complex) - c = StringIO() + c = BytesIO() np.savez(c, file_a=a, file_b=b) c.seek(0) l = np.load(c) @@ -228,95 +240,94 @@ class TestSaveTxt(TestCase): def test_array(self): a = np.array([[1, 2], [3, 4]], float) fmt = "%.18e" - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=fmt) c.seek(0) assert_equal(c.readlines(), - asbytes_nested( - [(fmt + ' ' + fmt + '\n') % (1, 2), - (fmt + ' ' + fmt + '\n') % (3, 4)])) + [asbytes((fmt + ' ' + fmt + '\n') % (1, 2)), + asbytes((fmt + ' ' + fmt + '\n') % (3, 4))]) a = np.array([[1, 2], [3, 4]], int) - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['1 2\n', '3 4\n'])) + assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) def test_1D(self): a = np.array([1, 2, 3, 4], int) - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) lines = c.readlines() - assert_equal(lines, asbytes_nested(['1\n', '2\n', '3\n', '4\n'])) + assert_equal(lines, [b'1\n', b'2\n', b'3\n', b'4\n']) def test_record(self): a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%d') c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['1 2\n', '3 4\n'])) + assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) def test_delimiter(self): a = np.array([[1., 2.], [3., 4.]]) - c = StringIO() - np.savetxt(c, a, delimiter=asbytes(','), fmt='%d') + c = BytesIO() + np.savetxt(c, a, delimiter=',', fmt='%d') c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['1,2\n', '3,4\n'])) + assert_equal(c.readlines(), [b'1,2\n', b'3,4\n']) def test_format(self): a = np.array([(1, 2), (3, 4)]) - c = StringIO() + c = BytesIO() # Sequence of formats np.savetxt(c, a, fmt=['%02d', '%3.1f']) c.seek(0) - assert_equal(c.readlines(), asbytes_nested(['01 2.0\n', '03 4.0\n'])) + assert_equal(c.readlines(), [b'01 2.0\n', b'03 4.0\n']) # A single multiformat string - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%02d : %3.1f') c.seek(0) lines = c.readlines() - assert_equal(lines, asbytes_nested(['01 : 2.0\n', '03 : 4.0\n'])) + assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n']) # Specify delimiter, should be overiden - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%02d : %3.1f', delimiter=',') c.seek(0) lines = c.readlines() - assert_equal(lines, asbytes_nested(['01 : 2.0\n', '03 : 4.0\n'])) + assert_equal(lines, [b'01 : 2.0\n', b'03 : 4.0\n']) def test_header_footer(self): """ Test the functionality of the header and footer keyword argument. """ - c = StringIO() + c = BytesIO() a = np.array([(1, 2), (3, 4)], dtype=np.int) test_header_footer = 'Test header / footer' # Test the header keyword argument np.savetxt(c, a, fmt='%1d', header=test_header_footer) c.seek(0) assert_equal(c.read(), - asbytes('# ' + test_header_footer +'\n1 2\n3 4\n' )) + asbytes('# ' + test_header_footer + '\n1 2\n3 4\n')) # Test the footer keyword argument - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt='%1d', footer=test_header_footer) c.seek(0) assert_equal(c.read(), asbytes('1 2\n3 4\n# ' + test_header_footer + '\n')) # Test the commentstr keyword argument used on the header - c = StringIO() + c = BytesIO() commentstr = '% ' - np.savetxt(c, a, fmt='%1d', header=test_header_footer, - comments=commentstr) + np.savetxt(c, a, fmt='%1d', + header=test_header_footer, comments=commentstr) c.seek(0) assert_equal(c.read(), asbytes(commentstr + test_header_footer + '\n' + '1 2\n3 4\n')) # Test the commentstr keyword argument used on the footer - c = StringIO() + c = BytesIO() commentstr = '% ' - np.savetxt(c, a, fmt='%1d', footer=test_header_footer, - comments=commentstr) + np.savetxt(c, a, fmt='%1d', + footer=test_header_footer, comments=commentstr) c.seek(0) assert_equal(c.read(), asbytes('1 2\n3 4\n' + commentstr + test_header_footer + '\n')) @@ -340,29 +351,29 @@ class TestSaveTxt(TestCase): im = np.e a[:] = re + 1.0j * im # One format only - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=' %+.3e') c.seek(0) lines = c.readlines() - _assert_floatstr_lines_equal(lines, asbytes_nested([ - ' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n', - ' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n'])) + _assert_floatstr_lines_equal(lines, + [b' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n', + b' ( +3.142e+00+ +2.718e+00j) ( +3.142e+00+ +2.718e+00j)\n']) # One format for each real and imaginary part - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=' %+.3e' * 2 * ncols) c.seek(0) lines = c.readlines() - _assert_floatstr_lines_equal(lines, asbytes_nested([ - ' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n', - ' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n'])) + _assert_floatstr_lines_equal(lines, + [b' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n', + b' +3.142e+00 +2.718e+00 +3.142e+00 +2.718e+00\n']) # One format for each complex number - c = StringIO() + c = BytesIO() np.savetxt(c, a, fmt=['(%.3e%+.3ej)'] * ncols) c.seek(0) lines = c.readlines() - _assert_floatstr_lines_equal(lines, asbytes_nested([ - '(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n', - '(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n'])) + _assert_floatstr_lines_equal(lines, + [b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n', + b'(3.142e+00+2.718e+00j) (3.142e+00+2.718e+00j)\n']) def _assert_floatstr_lines_equal(actual_lines, expected_lines): @@ -387,15 +398,15 @@ def _assert_floatstr_lines_equal(actual_lines, expected_lines): class TestLoadTxt(TestCase): def test_record(self): - c = StringIO() - c.write(asbytes('1 2\n3 4')) + c = TextIO() + c.write('1 2\n3 4') c.seek(0) x = np.loadtxt(c, dtype=[('x', np.int32), ('y', np.int32)]) a = np.array([(1, 2), (3, 4)], dtype=[('x', 'i4'), ('y', 'i4')]) assert_array_equal(x, a) - d = StringIO() - d.write(asbytes('M 64.0 75.0\nF 25.0 60.0')) + d = TextIO() + d.write('M 64.0 75.0\nF 25.0 60.0') d.seek(0) mydescriptor = {'names': ('gender', 'age', 'weight'), 'formats': ('S1', @@ -406,8 +417,8 @@ class TestLoadTxt(TestCase): assert_array_equal(y, b) def test_array(self): - c = StringIO() - c.write(asbytes('1 2\n3 4')) + c = TextIO() + c.write('1 2\n3 4') c.seek(0) x = np.loadtxt(c, dtype=int) @@ -420,23 +431,23 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_1D(self): - c = StringIO() - c.write(asbytes('1\n2\n3\n4\n')) + c = TextIO() + c.write('1\n2\n3\n4\n') c.seek(0) x = np.loadtxt(c, dtype=int) a = np.array([1, 2, 3, 4], int) assert_array_equal(x, a) - c = StringIO() - c.write(asbytes('1,2,3,4\n')) + c = TextIO() + c.write('1,2,3,4\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',') a = np.array([1, 2, 3, 4], int) assert_array_equal(x, a) def test_missing(self): - c = StringIO() - c.write(asbytes('1,2,3,,5\n')) + c = TextIO() + c.write('1,2,3,,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ converters={3:lambda s: int(s or - 999)}) @@ -444,8 +455,8 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_converters_with_usecols(self): - c = StringIO() - c.write(asbytes('1,2,3,,5\n6,7,8,9,10\n')) + c = TextIO() + c.write('1,2,3,,5\n6,7,8,9,10\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ converters={3:lambda s: int(s or - 999)}, \ @@ -454,8 +465,8 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_comments(self): - c = StringIO() - c.write(asbytes('# comment\n1,2,3,5\n')) + c = TextIO() + c.write('# comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ comments='#') @@ -463,16 +474,16 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_skiprows(self): - c = StringIO() - c.write(asbytes('comment\n1,2,3,5\n')) + c = TextIO() + c.write('comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ skiprows=1) a = np.array([1, 2, 3, 5], int) assert_array_equal(x, a) - c = StringIO() - c.write(asbytes('# comment\n1,2,3,5\n')) + c = TextIO() + c.write('# comment\n1,2,3,5\n') c.seek(0) x = np.loadtxt(c, dtype=int, delimiter=',', \ skiprows=1) @@ -481,14 +492,14 @@ class TestLoadTxt(TestCase): def test_usecols(self): a = np.array([[1, 2], [3, 4]], float) - c = StringIO() + c = BytesIO() np.savetxt(c, a) c.seek(0) x = np.loadtxt(c, dtype=float, usecols=(1,)) assert_array_equal(x, a[:, 1]) a = np.array([[1, 2, 3], [3, 4, 5]], float) - c = StringIO() + c = BytesIO() np.savetxt(c, a) c.seek(0) x = np.loadtxt(c, dtype=float, usecols=(1, 2)) @@ -503,16 +514,16 @@ class TestLoadTxt(TestCase): data = '''JOE 70.1 25.3 BOB 60.5 27.9 ''' - c = StringIO(data) + c = TextIO(data) names = ['stid', 'temp'] dtypes = ['S4', 'f8'] arr = np.loadtxt(c, usecols=(0, 2), dtype=zip(names, dtypes)) - assert_equal(arr['stid'], asbytes_nested(["JOE", "BOB"])) + assert_equal(arr['stid'], [b"JOE", b"BOB"]) assert_equal(arr['temp'], [25.3, 27.9]) def test_fancy_dtype(self): - c = StringIO() - c.write(asbytes('1,2,3.0\n4,5,6.0\n')) + c = TextIO() + c.write('1,2,3.0\n4,5,6.0\n') c.seek(0) dt = np.dtype([('x', int), ('y', [('t', int), ('s', float)])]) x = np.loadtxt(c, dtype=dt, delimiter=',') @@ -520,7 +531,7 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_shaped_dtype(self): - c = StringIO("aaaa 1.0 8.0 1 2 3 4 5 6") + c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6") dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ('block', int, (2, 3))]) x = np.loadtxt(c, dtype=dt) @@ -529,7 +540,7 @@ class TestLoadTxt(TestCase): assert_array_equal(x, a) def test_3d_shaped_dtype(self): - c = StringIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12") + c = TextIO("aaaa 1.0 8.0 1 2 3 4 5 6 7 8 9 10 11 12") dt = np.dtype([('name', 'S4'), ('x', float), ('y', float), ('block', int, (2, 2, 3))]) x = np.loadtxt(c, dtype=dt) @@ -543,7 +554,7 @@ class TestLoadTxt(TestCase): try: warnings.filterwarnings("ignore", message="loadtxt: Empty input file:") - c = StringIO() + c = TextIO() x = np.loadtxt(c) assert_equal(x.shape, (0,)) x = np.loadtxt(c, dtype=np.int64) @@ -554,8 +565,8 @@ class TestLoadTxt(TestCase): def test_unused_converter(self): - c = StringIO() - c.writelines([asbytes('1 21\n'), asbytes('3 42\n')]) + c = TextIO() + c.writelines(['1 21\n', '3 42\n']) c.seek(0) data = np.loadtxt(c, usecols=(1,), converters={0: lambda s: int(s, 16)}) @@ -570,12 +581,12 @@ class TestLoadTxt(TestCase): "Test using an explicit dtype with an object" from datetime import date import time - data = asbytes(""" 1; 2001-01-01 - 2; 2002-01-31 """) + data = """ 1; 2001-01-01 + 2; 2002-01-31 """ ndtype = [('idx', int), ('code', np.object)] func = lambda s: strptime(s.strip(), "%Y-%m-%d") converters = {1: func} - test = np.loadtxt(StringIO(data), delimiter=";", dtype=ndtype, + test = np.loadtxt(TextIO(data), delimiter=";", dtype=ndtype, converters=converters) control = np.array([(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))], dtype=ndtype) @@ -583,23 +594,23 @@ class TestLoadTxt(TestCase): def test_uint64_type(self): tgt = (9223372043271415339, 9223372043271415853) - c = StringIO() - c.write(asbytes("%s %s" % tgt)) + c = TextIO() + c.write("%s %s" % tgt) c.seek(0) res = np.loadtxt(c, dtype=np.uint64) assert_equal(res, tgt) def test_int64_type(self): tgt = (-9223372036854775807, 9223372036854775807) - c = StringIO() - c.write(asbytes("%s %s" % tgt)) + c = TextIO() + c.write("%s %s" % tgt) c.seek(0) res = np.loadtxt(c, dtype=np.int64) assert_equal(res, tgt) def test_universal_newline(self): f, name = mkstemp() - os.write(f, asbytes('1 21\r3 42\r')) + os.write(f, b'1 21\r3 42\r') os.close(f) try: @@ -609,29 +620,29 @@ class TestLoadTxt(TestCase): os.unlink(name) def test_empty_field_after_tab(self): - c = StringIO() - c.write(asbytes('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t')) + c = TextIO() + c.write('1 \t2 \t3\tstart \n4\t5\t6\t \n7\t8\t9.5\t') c.seek(0) dt = { 'names': ('x', 'y', 'z', 'comment'), 'formats': ('= 3: + from io import StringIO + else: + from StringIO import StringIO + dt = [("a", 'u1', 2), ("b", 'u1', 2)] x = np.loadtxt(StringIO("0 1 2 3"), dtype=dt) assert_equal(x, np.array([((0, 1), (2, 3))], dtype=dt)) diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 4b386e0b1..8cec1887b 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -1,10 +1,14 @@ from __future__ import division, absolute_import +import sys from numpy.testing import * import numpy.lib.utils as utils from numpy.lib import deprecate -from StringIO import StringIO +if sys.version_info[0] >= 3: + from io import StringIO +else: + from StringIO import StringIO def test_lookfor(): out = StringIO() diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 9a8776098..b81db681e 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -848,7 +848,12 @@ def _lookfor_generate_cache(module, import_modules, regenerate): global _lookfor_caches # Local import to speed up numpy's import time. import inspect - from cStringIO import StringIO + + if sys.version_info[0] >= 3: + # In Python3 stderr, stdout are text files. + from io import StringIO + else: + from StringIO import StringIO if module is None: module = "numpy" diff --git a/numpy/linalg/lapack_lite/clapack_scrub.py b/numpy/linalg/lapack_lite/clapack_scrub.py index 22f818e91..e6ac43e31 100644 --- a/numpy/linalg/lapack_lite/clapack_scrub.py +++ b/numpy/linalg/lapack_lite/clapack_scrub.py @@ -1,8 +1,8 @@ -#!/usr/bin/env python2.4 +#!/usr/bin/env python from __future__ import division, absolute_import import sys, os -from cStringIO import StringIO +from io import StringIO import re from Plex import * diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 3299262d1..6a37cf1b6 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -22,6 +22,23 @@ Released for unlimited redistribution. # pylint: disable-msg=E1002 from __future__ import division, absolute_import +import sys +import warnings + +import numpy as np +import numpy.core.umath as umath +import numpy.core.numerictypes as ntypes +from numpy import ndarray, amax, amin, iscomplexobj, bool_ +from numpy import array as narray +from numpy.lib.function_base import angle +from numpy.compat import getargspec, formatargspec +from numpy import expand_dims as n_expand_dims + +if sys.version_info[0] >= 3: + from functools import reduce + import pickle +else: + import cPickle as pickle __author__ = "Pierre GF Gerard-Marchant" __docformat__ = "restructuredtext en" @@ -69,23 +86,6 @@ __all__ = ['MAError', 'MaskError', 'MaskType', 'MaskedArray', 'var', 'where', 'zeros'] -import cPickle - -import numpy as np -from numpy import ndarray, amax, amin, iscomplexobj, bool_ -from numpy import array as narray - -import numpy.core.umath as umath -from numpy.lib.function_base import angle -import numpy.core.numerictypes as ntypes -from numpy.compat import getargspec, formatargspec -from numpy import expand_dims as n_expand_dims -import warnings - -import sys -if sys.version_info[0] >= 3: - from functools import reduce - MaskType = np.bool_ nomask = MaskType(0) @@ -7040,7 +7040,7 @@ def dump(a, F): """ if not hasattr(F, 'readline'): F = open(F, 'w') - return cPickle.dump(a, F) + return pickle.dump(a, F) def dumps(a): """ @@ -7055,7 +7055,7 @@ def dumps(a): returned. """ - return cPickle.dumps(a) + return pickle.dumps(a) def load(F): """ @@ -7079,7 +7079,7 @@ def load(F): """ if not hasattr(F, 'readline'): F = open(F, 'r') - return cPickle.load(F) + return pickle.load(F) def loads(strg): """ @@ -7097,7 +7097,7 @@ def loads(strg): dumps : Return a string corresponding to the pickling of a masked array. """ - return cPickle.loads(strg) + return pickle.loads(strg) ################################################################################ def fromfile(file, dtype=float, count= -1, sep=''): diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 383e4a907..7eb4bbed2 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -10,21 +10,20 @@ __author__ = "Pierre GF Gerard-Marchant" import types import warnings +import sys +import pickle import numpy as np +import numpy.ma.core import numpy.core.fromnumeric as fromnumeric from numpy import ndarray from numpy.ma.testutils import * - -import numpy.ma.core from numpy.ma.core import * - from numpy.compat import asbytes, asbytes_nested from numpy.testing.utils import WarningManager pi = np.pi -import sys if sys.version_info[0] >= 3: from functools import reduce @@ -369,20 +368,18 @@ class TestMaskedArray(TestCase): def test_pickling(self): "Tests pickling" - import cPickle a = arange(10) a[::3] = masked a.fill_value = 999 - a_pickled = cPickle.loads(a.dumps()) + a_pickled = pickle.loads(a.dumps()) assert_equal(a_pickled._mask, a._mask) assert_equal(a_pickled._data, a._data) assert_equal(a_pickled.fill_value, 999) def test_pickling_subbaseclass(self): "Test pickling w/ a subclass of ndarray" - import cPickle a = array(np.matrix(list(range(10))), mask=[1, 0, 1, 0, 0] * 2) - a_pickled = cPickle.loads(a.dumps()) + a_pickled = pickle.loads(a.dumps()) assert_equal(a_pickled._mask, a._mask) assert_equal(a_pickled, a) self.assertTrue(isinstance(a_pickled._data, np.matrix)) @@ -390,37 +387,28 @@ class TestMaskedArray(TestCase): def test_pickling_maskedconstant(self): "Test pickling MaskedConstant" - import cPickle mc = np.ma.masked - mc_pickled = cPickle.loads(mc.dumps()) + mc_pickled = pickle.loads(mc.dumps()) assert_equal(mc_pickled._baseclass, mc._baseclass) assert_equal(mc_pickled._mask, mc._mask) assert_equal(mc_pickled._data, mc._data) def test_pickling_wstructured(self): "Tests pickling w/ structured array" - import cPickle a = array([(1, 1.), (2, 2.)], mask=[(0, 0), (0, 1)], dtype=[('a', int), ('b', float)]) - a_pickled = cPickle.loads(a.dumps()) + a_pickled = pickle.loads(a.dumps()) assert_equal(a_pickled._mask, a._mask) assert_equal(a_pickled, a) def test_pickling_keepalignment(self): "Tests pickling w/ F_CONTIGUOUS arrays" - import cPickle a = arange(10) a.shape = (-1, 2) b = a.T - test = cPickle.loads(cPickle.dumps(b)) + test = pickle.loads(pickle.dumps(b)) assert_equal(test, b) -# def test_pickling_oddity(self): -# "Test some pickling oddity" -# import cPickle -# a = array([{'a':1}, {'b':2}, 3], dtype=object) -# test = cPickle.loads(cPickle.dumps(a)) -# assert_equal(test, a) def test_single_element_subscript(self): "Tests single element subscripts of Maskedarrays." diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py index d6f8d9765..3b9296a04 100644 --- a/numpy/ma/tests/test_mrecords.py +++ b/numpy/ma/tests/test_mrecords.py @@ -7,30 +7,30 @@ """ from __future__ import division, absolute_import -__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" -__revision__ = "$Revision: 3473 $" -__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' - import sys +import warnings +import pickle + import numpy as np +import numpy.ma.testutils +import numpy.ma as ma from numpy import recarray from numpy.core.records import fromrecords as recfromrecords, \ fromarrays as recfromarrays from numpy.compat import asbytes, asbytes_nested - -import numpy.ma.testutils from numpy.ma.testutils import * - -import numpy.ma as ma from numpy.ma import masked, nomask - -import warnings from numpy.testing.utils import WarningManager - from numpy.ma.mrecords import MaskedRecords, mrecarray, fromarrays, \ fromtextfile, fromrecords, addfield + +__author__ = "Pierre GF Gerard-Marchant ($Author: jarrod.millman $)" +__revision__ = "$Revision: 3473 $" +__date__ = '$Date: 2007-10-29 17:18:13 +0200 (Mon, 29 Oct 2007) $' + + #.............................................................................. class TestMRecords(TestCase): "Base test class for MaskedArrays." @@ -293,11 +293,10 @@ class TestMRecords(TestCase): # def test_pickling(self): "Test pickling" - import cPickle base = self.base.copy() mrec = base.view(mrecarray) - _ = cPickle.dumps(mrec) - mrec_ = cPickle.loads(_) + _ = pickle.dumps(mrec) + mrec_ = pickle.loads(_) assert_equal(mrec_.dtype, mrec.dtype) assert_equal_records(mrec_._data, mrec._data) assert_equal(mrec_._mask, mrec._mask) diff --git a/numpy/numarray/functions.py b/numpy/numarray/functions.py index 2492d5f3f..9da96009f 100644 --- a/numpy/numarray/functions.py +++ b/numpy/numarray/functions.py @@ -25,7 +25,6 @@ __all__ += ['vdot', 'dot', 'matrixmultiply', 'ravel', 'indices', ] import copy -import copy_reg import types import os import sys @@ -44,6 +43,8 @@ from .numerictypes import typefrom if sys.version_info[0] >= 3: import copyreg as copy_reg +else: + import copy_reg isBigEndian = sys.byteorder != 'little' value = tcode = 'f' diff --git a/numpy/oldnumeric/compat.py b/numpy/oldnumeric/compat.py index 46c3c727f..b63226571 100644 --- a/numpy/oldnumeric/compat.py +++ b/numpy/oldnumeric/compat.py @@ -3,6 +3,17 @@ """ from __future__ import division, absolute_import +import sys +import copy +import pickle +from pickle import dump, dumps + +import numpy.core.multiarray as multiarray +import numpy.core.umath as um +from numpy.core.numeric import array +from . import functions + + __all__ = ['NewAxis', 'UFuncType', 'UfuncType', 'ArrayType', 'arraytype', 'LittleEndian', 'arrayrange', 'matrixmultiply', @@ -13,13 +24,6 @@ __all__ = ['NewAxis', 'Unpickler', 'Pickler' ] -import numpy.core.multiarray as multiarray -import numpy.core.umath as um -from numpy.core.numeric import array -from . import functions -import sys - -from cPickle import dump, dumps mu = multiarray @@ -47,8 +51,7 @@ def DumpArray(m, fp): m.dump(fp) def LoadArray(fp): - import cPickle - return cPickle.load(fp) + return pickle.load(fp) def array_constructor(shape, typecode, thestr, Endian=LittleEndian): if typecode == "O": @@ -70,8 +73,7 @@ def pickle_array(a): (a.shape, a.dtype.char, a.tostring(), LittleEndian)) def loads(astr): - import cPickle - arr = cPickle.loads(astr.replace('Numeric', 'numpy.oldnumeric')) + arr = pickle.loads(astr.replace('Numeric', 'numpy.oldnumeric')) return arr def load(fp): @@ -97,7 +99,6 @@ def _LoadArray(fp): else: return m -import pickle, copy if sys.version_info[0] >= 3: class Unpickler(pickle.Unpickler): # XXX: should we implement this? It's not completely straightforward diff --git a/numpy/oldnumeric/misc.py b/numpy/oldnumeric/misc.py index 8f1e1ae96..5ef1f0ac9 100644 --- a/numpy/oldnumeric/misc.py +++ b/numpy/oldnumeric/misc.py @@ -14,18 +14,17 @@ __all__ = ['sort', 'copy_reg', 'clip', 'rank', 'dot', 'outerproduct', 'innerproduct', 'insert'] import types -import StringIO import pickle import math import copy -import copy_reg - import sys + if sys.version_info[0] >= 3: - import copyreg - import io - StringIO = io.BytesIO - copy_reg = copyreg + import copyreg as copy_reg + from io import BytesIO as StringIO +else: + import copy_reg + from StringIO import StringIO from numpy import sort, clip, rank, sign, shape, putmask, allclose, size,\ choose, swapaxes, array_str, array_repr, e, pi, put, \ diff --git a/numpy/testing/utils.py b/numpy/testing/utils.py index c08dd0c33..62046814f 100644 --- a/numpy/testing/utils.py +++ b/numpy/testing/utils.py @@ -12,6 +12,11 @@ import types import warnings from .nosetester import import_nose +if sys.version_info[0] >= 3: + from io import StringIO +else: + from StringIO import StringIO + __all__ = ['assert_equal', 'assert_almost_equal','assert_approx_equal', 'assert_array_equal', 'assert_array_less', 'assert_string_equal', 'assert_array_almost_equal', 'assert_raises', 'build_err_msg', @@ -346,8 +351,7 @@ def print_assert_equal(test_string,actual,desired): import pprint if not (actual == desired): - import cStringIO - msg = cStringIO.StringIO() + msg = StringIO() msg.write(test_string) msg.write(' failed\nACTUAL: \n') pprint.pprint(actual,msg) diff --git a/setup.py b/setup.py index b30433691..0ba707462 100755 --- a/setup.py +++ b/setup.py @@ -23,10 +23,10 @@ import sys import re import subprocess -if sys.version_info[0] < 3: - import __builtin__ as builtins -else: +if sys.version_info[0] >= 3: import builtins +else: + import __builtin__ as builtins CLASSIFIERS = """\ Development Status :: 5 - Production/Stable diff --git a/tools/py3tool.py b/tools/py3tool.py index 656327450..d7fc94240 100755 --- a/tools/py3tool.py +++ b/tools/py3tool.py @@ -77,7 +77,8 @@ FIXES_TO_SKIP = [ 'raw_input', 'xreadlines', 'xrange', - 'import' + 'import', + 'imports' ] skip_fixes= [] diff --git a/tools/win32build/build.py b/tools/win32build/build.py index 43d8a395d..dc12e807d 100644 --- a/tools/win32build/build.py +++ b/tools/win32build/build.py @@ -88,8 +88,12 @@ def move_binary(arch, pyver): os.path.join("binaries", get_binary_name(arch))) def get_numpy_version(): - import __builtin__ - __builtin__.__NUMPY_SETUP__ = True + if sys.version_info[0] >= 3: + import builtins + else: + import __builtin__ as builtins + + builtins.__NUMPY_SETUP__ = True from numpy.version import version return version -- cgit v1.2.1 From 4394515cd5632a7f110993ff75033d407d10861d Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Tue, 2 Apr 2013 13:47:32 -0600 Subject: BUG: Fix stray '.' in import statement. There was a stray period at the end of an import statement in `doc/cdoc/numpyfilter.py`. Looks like a cut and paste error that was fixed elsewhere but escaped there because the module isn't tested. A search shows that this was the only spot in which the error was still present. --- doc/cdoc/numpyfilter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/cdoc/numpyfilter.py b/doc/cdoc/numpyfilter.py index 7137df2d2..d5021f92a 100755 --- a/doc/cdoc/numpyfilter.py +++ b/doc/cdoc/numpyfilter.py @@ -17,7 +17,7 @@ import optparse if sys.version_info[0] >= 3: import pickle else: - import cPickle as pickle. + import cPickle as pickle CACHE_FILE = 'build/rst-cache.pck' @@ -47,7 +47,7 @@ def filter_comment(text): if text.startswith('UFUNC_API'): text = text[9:].strip() - html = render_html(text) + html = render_html(text) return html def process_match(m, cache=None): -- cgit v1.2.1