diff options
author | Giampaolo Rodola <g.rodola@gmail.com> | 2020-02-12 04:20:20 -0800 |
---|---|---|
committer | Giampaolo Rodola <g.rodola@gmail.com> | 2020-02-12 04:20:20 -0800 |
commit | 15a546e0376029d69684875adbb8c9eb2c44af8a (patch) | |
tree | 6ad6cd6b9019682e3d97713704930495c6d85c05 | |
parent | 135cce8dfdd678e06aec3d0c06c8a25a1b8cd2d4 (diff) | |
parent | 205f213dd5f548d19873a55ee1a7dc28a77d46e1 (diff) | |
download | psutil-15a546e0376029d69684875adbb8c9eb2c44af8a.tar.gz |
merge from master
-rw-r--r-- | HISTORY.rst | 2 | ||||
-rw-r--r-- | Makefile | 2 | ||||
-rw-r--r-- | README.rst | 2 | ||||
-rw-r--r-- | docs/DEVGUIDE.rst | 29 | ||||
-rw-r--r-- | docs/Makefile | 2 | ||||
-rw-r--r-- | docs/_static/css/custom.css | 22 | ||||
-rw-r--r-- | docs/index.rst | 160 | ||||
-rw-r--r-- | psutil/_common.py | 8 | ||||
-rw-r--r-- | psutil/_psutil_common.c | 48 | ||||
-rw-r--r-- | psutil/_psutil_common.h | 18 | ||||
-rw-r--r-- | psutil/_psutil_windows.c | 18 | ||||
-rw-r--r-- | psutil/_pswindows.py | 19 | ||||
-rw-r--r-- | psutil/arch/windows/cpu.c | 12 | ||||
-rw-r--r-- | psutil/arch/windows/wmi.c | 2 | ||||
-rw-r--r-- | psutil/tests/__init__.py | 6 | ||||
-rwxr-xr-x | psutil/tests/runner.py | 6 | ||||
-rwxr-xr-x | psutil/tests/test_linux.py | 3 | ||||
-rwxr-xr-x | psutil/tests/test_system.py | 11 | ||||
-rw-r--r-- | psutil/tests/test_unicode.py | 36 | ||||
-rwxr-xr-x | psutil/tests/test_windows.py | 25 | ||||
-rwxr-xr-x | scripts/internal/winmake.py | 12 |
21 files changed, 218 insertions, 225 deletions
diff --git a/HISTORY.rst b/HISTORY.rst index 68309241..233236cc 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,7 +18,7 @@ XXXX-XX-XX raising AccessDenied). - 1679_: [Windows] net_connections() and Process.connections() are 10% faster. - 1681_: [Linux] disk_partitions() now also shows swap partitions. - +- 1686_: [Windows] added support for PyPy on Windows. **Bug fixes** @@ -17,7 +17,7 @@ DEPS = \ pyperf \ requests \ setuptools \ - sphinx \ + sphinx==2.2.2 \ twine \ unittest2 \ virtualenv \ @@ -100,7 +100,7 @@ psutil currently supports the following platforms: - **Sun Solaris** - **AIX** -...both **32-bit** and **64-bit** architectures. Supported Python versions are **2.6**, **2.7** and **3.4+**. `PyPy <http://pypy.org/>`__ is also known to work. +...both **32-bit** and **64-bit** architectures. Supported Python versions are **2.6**, **2.7** and **3.4+**. `PyPy3 <http://pypy.org/>`__ is also known to work. psutil for enterprise ===================== diff --git a/docs/DEVGUIDE.rst b/docs/DEVGUIDE.rst index 598c8b61..e07d977e 100644 --- a/docs/DEVGUIDE.rst +++ b/docs/DEVGUIDE.rst @@ -1,24 +1,13 @@ -Setup and running tests -======================= - -If you plan on hacking on psutil this is what you're supposed to do first: +Build, setup and running tests +=============================== -- clone the GIT repository: - -.. code-block:: bash - - $ git clone git@github.com:giampaolo/psutil.git - -- install test deps and GIT hooks: +Make sure to `install <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`__ +a C compiler first, then: .. code-block:: bash + git clone git@github.com:giampaolo/psutil.git make setup-dev-env - -- run tests: - -.. code-block:: bash - make test - bear in mind that ``make``(see `Makefile`_) is the designated tool to run @@ -60,13 +49,7 @@ On Windows: .. code-block:: bat - set PYTHON=C:\python35\python.exe && make test - -...or: - -.. code-block:: bat - - make -p 35 test + make -p C:\python35\python.exe test If you want to modify psutil and run a script on the fly which uses it do (on UNIX): diff --git a/docs/Makefile b/docs/Makefile index 0c4bdf48..c7f4723a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -PYTHON = python +PYTHON = python3 SPHINXOPTS = SPHINXBUILD = $(PYTHON) -m sphinx PAPER = diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index b76f442a..c5c201e4 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -15,10 +15,30 @@ border-right:10px !important; } -.local-toc li ul li{ +.local-toc li ul li { padding-left: 20px !important; } +.rst-content ul p { + margin-bottom: 0px !important; +} + +.document td { + padding-bottom: 0px !important; +} + +.document th { + padding-top: 0px !important; + padding-bottom: 0px !important; +} + +.document th p { + margin-bottom: 0px !important; +} + +.document th p { +} + .function .descclassname { font-weight: normal !important; } diff --git a/docs/index.rst b/docs/index.rst index 97dce444..e60058cd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2345,46 +2345,6 @@ Hardware constants >>> if psutil.version_info >= (4, 5): ... pass ----- - -Unicode -======= - -Starting from version 5.3.0 psutil adds unicode support, see `issue #1040`_. -The notes below apply to *any* API returning a string such as -:meth:`Process.exe` or :meth:`Process.cwd`, including non-filesystem related -methods such as :meth:`Process.username` or :meth:`WindowsService.description`: - -* all strings are encoded by using the OS filesystem encoding - (``sys.getfilesystemencoding()``) which varies depending on the platform - (e.g. "UTF-8" on macOS, "mbcs" on Win) -* no API call is supposed to crash with ``UnicodeDecodeError`` -* instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the corrupted characters in the string: - * Python 3: ``sys.getfilesystemencodeerrors()`` (PY 3.6+) or - ``"surrogatescape"`` on POSIX and ``"replace"`` on Windows - * Python 2: ``"replace"`` -* on Python 2 all APIs return bytes (``str`` type), never ``unicode`` -* on Python 2, you can go back to ``unicode`` by doing: - -.. code-block:: python - - >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") - -Example which filters processes with a funky name working with both Python 2 -and 3:: - - # -*- coding: utf-8 -*- - import psutil, sys - - PY3 = sys.version_info[0] == 2 - LOOKFOR = u"ƒőő" - for proc in psutil.process_iter(attrs=['name']): - name = proc.info['name'] - if not PY3: - name = unicode(name, sys.getdefaultencoding(), errors="replace") - if LOOKFOR == name: - print("process %s found" % p) - Recipes ======= @@ -2450,61 +2410,14 @@ Kill process tree callback=on_terminate) return (gone, alive) -Terminate my children ---------------------- - -This may be useful in unit tests whenever sub-processes are started. -This will help ensure that no extra children (zombies) stick around to hog -resources. - -:: - - import psutil - - def reap_children(timeout=3): - "Tries hard to terminate and ultimately kill all the children of this process." - def on_terminate(proc): - print("process {} terminated with exit code {}".format(proc, proc.returncode)) - - procs = psutil.Process().children() - # send SIGTERM - for p in procs: - try: - p.terminate() - except psutil.NoSuchProcess: - pass - gone, alive = psutil.wait_procs(procs, timeout=timeout, callback=on_terminate) - if alive: - # send SIGKILL - for p in alive: - print("process {} survived SIGTERM; trying SIGKILL".format(p)) - try: - p.kill() - except psutil.NoSuchProcess: - pass - gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate) - if alive: - # give up - for p in alive: - print("process {} survived SIGKILL; giving up".format(p)) - Filtering and sorting processes ------------------------------- -This is a collection of one-liners showing how to use :func:`process_iter()` in -order to filter for processes and sort them. - -Setup:: +A collection of code samples showing how to use :func:`process_iter()` to filter processes and sort them. Setup:: >>> import psutil >>> from pprint import pprint as pp -Processes having "python" in their name:: - - >>> pp([p.info for p in psutil.process_iter(attrs=['pid', 'name']) if 'python' in p.info['name']]) - [{'name': 'python3', 'pid': 21947}, - {'name': 'python', 'pid': 23835}] - Processes owned by user:: >>> import getpass @@ -2522,11 +2435,9 @@ Processes actively running:: Processes using log files:: - >>> import os - >>> import psutil >>> for p in psutil.process_iter(attrs=['name', 'open_files']): ... for file in p.info['open_files'] or []: - ... if os.path.splitext(file.path)[1] == '.log': + ... if file.path.endswith('.log'): ... print("%-5s %-10s %s" % (p.pid, p.info['name'][:10], file.path)) ... 1510 upstart /home/giampaolo/.cache/upstart/unity-settings-daemon.log @@ -2540,13 +2451,6 @@ Processes consuming more than 500M of memory:: (3038, 'chrome', 1120088064), (21915, 'sublime_text', 615407616)] -Top 3 most memory consuming processes:: - - >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'memory_percent']), key=lambda p: p.info['memory_percent'])][-3:]) - [(21915, {'memory_percent': 3.6815453247662737, 'name': 'sublime_text'}), - (3038, {'memory_percent': 6.732935429979187, 'name': 'chrome'}), - (3249, {'memory_percent': 8.994554843376399, 'name': 'chrome'})] - Top 3 processes which consumed the most CPU time:: >>> pp([(p.pid, p.info['name'], sum(p.info['cpu_times'])) for p in sorted(psutil.process_iter(attrs=['name', 'cpu_times']), key=lambda p: sum(p.info['cpu_times'][:2]))][-3:]) @@ -2554,20 +2458,6 @@ Top 3 processes which consumed the most CPU time:: (1150, 'Xorg', 11116.989999999998), (2650, 'chrome', 18451.97)] -Top 3 processes which caused the most I/O:: - - >>> pp([(p.pid, p.info['name']) for p in sorted(psutil.process_iter(attrs=['name', 'io_counters']), key=lambda p: p.info['io_counters'] and p.info['io_counters'][:2])][-3:]) - [(21915, 'sublime_text'), - (1871, 'pulseaudio'), - (1510, 'upstart')] - -Top 3 processes opening more file descriptors:: - - >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'num_fds']), key=lambda p: p.info['num_fds'])][-3:]) - [(21915, {'name': 'sublime_text', 'num_fds': 105}), - (2721, {'name': 'chrome', 'num_fds': 185}), - (2650, {'name': 'chrome', 'num_fds': 354})] - Bytes conversion ---------------- @@ -2600,27 +2490,6 @@ Bytes conversion 100399730688 93.5G -Supported platforms -=================== - -These are the platforms I develop and test on: - -* Linux Ubuntu 18.04 -* Windows 10 (support back to Windows Vista) -* MacOS 10.11 El Captain -* Solaris 10 -* FreeBSD 11 -* OpenBSD 6.4 -* NetBSD 8.0 -* AIX 6.1 TL8 (maintainer `Arnon Yaari <https://github.com/wiggin15>`__) - -Earlier versions are supposed to work but are not tested. -For Linux, Windows and MacOS we have continuos integration. Other platforms -are tested manually from time to time. -Minimum supported Windows version is Windows Vista (Windows XP and Windows -Server 2003 are not supported). -Supported Python versions are 3.4+, 2.7 and 2.6. - FAQs ==== @@ -2638,21 +2507,30 @@ FAQs Running tests ============= -There are two ways of running tests. If psutil is already installed use:: - - $ python -m psutil.tests - -You can use this method as a quick way to make sure psutil fully works on your -platform. If you have a copy of the source code you can also use:: +:: - $ make test + $ python3 -m psutil.tests Development guide ================= -If you plan on hacking on psutil (e.g. want to add a new feature or fix a bug) +If you want to hacking on psutil (e.g. want to add a new feature or fix a bug) take a look at the `development guide`_. +Platforms support history +========================= + +* psutil 5.7.0 (2020-02): drop Windows XP & Server 2003 support +* psutil 5.7.0 (2020-02): **PyPy** on Windows +* psutil 5.4.0 (2017-11): **AIX** +* psutil 3.4.1 (2016-01): **NetBSD** +* psutil 3.3.0 (2015-11): **OpenBSD** +* psutil 1.0.0 (2013-07): **Solaris** +* psutil 0.1.1 (2009-03): **FreeBSD** +* psutil 0.1.0 (2009-01): **Linux, Windows, macOS** + +Supported Python versions are 2.6, 2.7, 3.4+ and PyPy3. + Timeline ======== diff --git a/psutil/_common.py b/psutil/_common.py index 453c771d..9306cd15 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -790,8 +790,14 @@ def hilite(s, ok=True, bold=False): if bool(os.getenv('PSUTIL_DEBUG', 0)): + import inspect + def debug(msg): - print("psutil-debug> " + msg, file=sys.stderr) + """If PSUTIL_DEBUG env var is set, print a debug message to stderr.""" + fname, lineno, func_name, lines, index = inspect.getframeinfo( + inspect.currentframe().f_back) + print("psutil-debug [%s:%s]> %s" % (fname, lineno, msg), + file=sys.stderr) else: def debug(msg): pass diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 990d59a6..b8c6b5e5 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -10,17 +10,15 @@ #include "_psutil_common.h" // ==================================================================== -// --- Global vars / constants +// --- Global vars // ==================================================================== - int PSUTIL_DEBUG = 0; int PSUTIL_TESTING = 0; // PSUTIL_CONN_NONE - // ==================================================================== -// --- Python functions and backward compatibility +// --- Custom exceptions // ==================================================================== /* @@ -45,10 +43,6 @@ PyErr_SetFromOSErrnoWithSyscall(const char *syscall) { } -// ==================================================================== -// --- Custom exceptions -// ==================================================================== - /* * Set OSError(errno=ESRCH, strerror="No such process (originated from") * Python exception. @@ -133,6 +127,7 @@ psutil_setup(void) { // --- Windows // ==================================================================== + #ifdef PSUTIL_WINDOWS #include <windows.h> @@ -149,6 +144,43 @@ CRITICAL_SECTION PSUTIL_CRITICAL_SECTION; #define WIN32_FROM_NTSTATUS(Status) (((ULONG)(Status)) & 0xffff) +// PyPy on Windows +#if defined(PYPY_VERSION) && !defined(PyErr_SetFromWindowsErrWithFilename) +PyObject * +PyErr_SetFromWindowsErrWithFilename(int winerr, const char *filename) { + PyObject *py_exc = NULL; + PyObject *py_winerr = NULL; + + if (winerr == 0) + winerr = GetLastError(); + if (filename == NULL) { + py_exc = PyObject_CallFunction(PyExc_OSError, "(is)", winerr, + strerror(winerr)); + } + else { + py_exc = PyObject_CallFunction(PyExc_OSError, "(iss)", winerr, + strerror(winerr), filename); + } + if (py_exc == NULL) + return NULL; + + py_winerr = Py_BuildValue("i", winerr); + if (py_winerr == NULL) + goto error; + if (PyObject_SetAttrString(py_exc, "winerror", py_winerr) != 0) + goto error; + PyErr_SetObject(PyExc_OSError, py_exc); + Py_XDECREF(py_exc); + return NULL; + +error: + Py_XDECREF(py_exc); + Py_XDECREF(py_winerr); + return NULL; +} +#endif + + // A wrapper around GetModuleHandle and GetProcAddress. PVOID psutil_GetProcAddress(LPCSTR libname, LPCSTR procname) { diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 2fccab81..b072e357 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -16,7 +16,7 @@ extern int PSUTIL_DEBUG; static const int PSUTIL_CONN_NONE = 128; // ==================================================================== -// --- Python functions and backward compatibility +// --- Backward compatibility with missing Python.h APIs // ==================================================================== #if PY_MAJOR_VERSION < 3 @@ -32,7 +32,7 @@ static const int PSUTIL_CONN_NONE = 128; // SIZEOF_INT|LONG is missing on Linux + PyPy (only?). // SIZEOF_PID_T is missing on Windows + Python2. // In we can't determine pid_t size we assume it's an (int). -// On major UNIX platforms I've seen pid_t is treated as int. +// On all UNIX platforms I've seen pid_t is defined as an int. // _getpid() on Windows returns an int. We can't be 100% sure though, // (in that case we'd probably get compiler warnings). #if !defined(SIZEOF_INT) @@ -60,9 +60,14 @@ static const int PSUTIL_CONN_NONE = 128; #endif #endif -#if PY_MAJOR_VERSION < 3 +// Python 2 or PyPy on Windows +#ifndef PyLong_FromPid #if ((SIZEOF_PID_T == SIZEOF_INT) || (SIZEOF_PID_T == SIZEOF_LONG)) - #define PyLong_FromPid PyInt_FromLong + #if PY_MAJOR_VERSION >= 3 + #define PyLong_FromPid PyLong_FromLong + #else + #define PyLong_FromPid PyInt_FromLong + #endif #elif defined(SIZEOF_LONG_LONG) && SIZEOF_PID_T == SIZEOF_LONG_LONG #define PyLong_FromPid PyLong_FromLongLong #else @@ -122,4 +127,9 @@ int psutil_setup(void); PVOID psutil_GetProcAddress(LPCSTR libname, LPCSTR procname); PVOID psutil_GetProcAddressFromLib(LPCSTR libname, LPCSTR procname); PVOID psutil_SetFromNTStatusErr(NTSTATUS Status, const char *syscall); + + #if defined(PYPY_VERSION) && !defined(PyErr_SetFromWindowsErrWithFilename) + PyObject *PyErr_SetFromWindowsErrWithFilename(int ierr, + const char *filename); + #endif #endif diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 102b9e3d..f9405707 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -228,8 +228,10 @@ psutil_proc_wait(PyObject *self, PyObject *args) { // return None instead. Py_RETURN_NONE; } - else - return PyErr_SetFromWindowsErr(0); + else { + PyErr_SetFromWindowsErr(0); + return NULL; + } } // wait until the process has terminated @@ -698,8 +700,10 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { MEMORYSTATUSEX memInfo; memInfo.dwLength = sizeof(MEMORYSTATUSEX); - if (! GlobalMemoryStatusEx(&memInfo)) - return PyErr_SetFromWindowsErr(0); + if (! GlobalMemoryStatusEx(&memInfo)) { + PyErr_SetFromWindowsErr(0); + return NULL; + } return Py_BuildValue("(LLLLLL)", memInfo.ullTotalPhys, // total memInfo.ullAvailPhys, // avail @@ -1571,8 +1575,10 @@ static PyObject * psutil_sensors_battery(PyObject *self, PyObject *args) { SYSTEM_POWER_STATUS sps; - if (GetSystemPowerStatus(&sps) == 0) - return PyErr_SetFromWindowsErr(0); + if (GetSystemPowerStatus(&sps) == 0) { + PyErr_SetFromWindowsErr(0); + return NULL; + } return Py_BuildValue( "iiiI", sps.ACLineStatus, // whether AC is connected: 0=no, 1=yes, 255=unknown diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 83793c5a..d8abf2e6 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -17,6 +17,7 @@ from . import _common from ._common import AccessDenied from ._common import conn_tmap from ._common import conn_to_ntuple +from ._common import debug from ._common import ENCODING from ._common import ENCODING_ERRS from ._common import isfile_strict @@ -80,7 +81,7 @@ __extra__all__ = [ CONN_DELETE_TCB = "DELETE_TCB" ERROR_PARTIAL_COPY = 299 - +PYPY = '__pypy__' in sys.builtin_module_names if enum is None: AF_LINK = -1 @@ -752,7 +753,18 @@ class Process(object): @wrap_exceptions @memoize_when_activated def exe(self): - exe = cext.proc_exe(self.pid) + if PYPY: + try: + exe = cext.proc_exe(self.pid) + except WindowsError as err: + # 24 = ERROR_TOO_MANY_OPEN_FILES. Not sure why this happens + # (perhaps PyPy's JIT delaying garbage collection of files?). + if err.errno == 24: + debug("%r forced into AccessDenied" % err) + raise AccessDenied(self.pid, self._name) + raise + else: + exe = cext.proc_exe(self.pid) if not PY3: exe = py2_strencode(exe) if exe.startswith('\\'): @@ -916,9 +928,6 @@ class Process(object): @wrap_exceptions def create_time(self): - # special case for kernel process PIDs; return system boot time - if self.pid in (0, 4): - return boot_time() try: return cext.proc_create_time(self.pid) except OSError as err: diff --git a/psutil/arch/windows/cpu.c b/psutil/arch/windows/cpu.c index 9a22e149..18f32e59 100644 --- a/psutil/arch/windows/cpu.c +++ b/psutil/arch/windows/cpu.c @@ -50,8 +50,10 @@ psutil_cpu_times(PyObject *self, PyObject *args) { double idle, kernel, user, system; FILETIME idle_time, kernel_time, user_time; - if (!GetSystemTimes(&idle_time, &kernel_time, &user_time)) - return PyErr_SetFromWindowsErr(0); + if (!GetSystemTimes(&idle_time, &kernel_time, &user_time)) { + PyErr_SetFromWindowsErr(0); + return NULL; + } idle = (double)((HI_T * idle_time.dwHighDateTime) + \ (LO_T * idle_time.dwLowDateTime)); @@ -384,8 +386,10 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { // Allocate size. size = ncpus * sizeof(PROCESSOR_POWER_INFORMATION); pBuffer = (BYTE*)LocalAlloc(LPTR, size); - if (! pBuffer) - return PyErr_SetFromWindowsErr(0); + if (! pBuffer) { + PyErr_SetFromWindowsErr(0); + return NULL; + } // Syscall. ret = CallNtPowerInformation( diff --git a/psutil/arch/windows/wmi.c b/psutil/arch/windows/wmi.c index b790c08e..42a70df7 100644 --- a/psutil/arch/windows/wmi.c +++ b/psutil/arch/windows/wmi.c @@ -96,7 +96,7 @@ psutil_init_loadavg_counter(PyObject *self, PyObject *args) { Py_RETURN_NONE; error: - PyErr_SetExcFromWindowsErr(PyExc_OSError, 0); + PyErr_SetFromWindowsErr(0); return NULL; } diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 8a373386..3e4dc880 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -118,7 +118,6 @@ TRAVIS = bool(os.environ.get('TRAVIS')) APPVEYOR = bool(os.environ.get('APPVEYOR')) CIRRUS = bool(os.environ.get('CIRRUS')) CI_TESTING = TRAVIS or APPVEYOR or CIRRUS -PYPY = '__pypy__' in sys.builtin_module_names # --- configurable defaults @@ -1111,9 +1110,12 @@ else: ext = ".dll" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if - os.path.splitext(x.path)[1].lower() == ext and + x.path.lower().endswith(ext) and 'python' in os.path.basename(x.path).lower() and 'wow64' not in x.path.lower()] + if PYPY and not libs: + libs = [x.path for x in psutil.Process().memory_maps() if + 'pypy' in os.path.basename(x.path).lower()] src = random.choice(libs) shutil.copyfile(src, dst) cfile = None diff --git a/psutil/tests/runner.py b/psutil/tests/runner.py index a10c6413..f8601bad 100755 --- a/psutil/tests/runner.py +++ b/psutil/tests/runner.py @@ -177,11 +177,7 @@ def save_failed_tests(result): with open(FAILED_TESTS_FNAME, 'wt') as f: for t in result.errors + result.failures: tname = str(t[0]) - print(tname) - try: - unittest.defaultTestLoader.loadTestsFromName(tname) - except Exception: - import pdb; pdb.set_trace() + unittest.defaultTestLoader.loadTestsFromName(tname) f.write(tname + '\n') diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4736bd6b..5a48a445 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1070,6 +1070,9 @@ class TestSystemDiskPartitions(unittest.TestCase): @unittest.skipIf(not os.path.exists('/proc/swaps'), "/proc/swaps not available") def test_swap(self): + with open('/proc/swaps') as f: + if not f.readline() or not f.readlines(): + raise self.skipTest("/proc/swaps is empty") types = [x.fstype for x in psutil.disk_partitions(all=False)] self.assertNotIn('swap', types) types = [x.fstype for x in psutil.disk_partitions(all=True)] diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index c6f8a17a..2d606be4 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -31,9 +31,9 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import FileNotFoundError from psutil._compat import long -from psutil.tests import CI_TESTING from psutil.tests import ASCII_FS from psutil.tests import check_net_address +from psutil.tests import CI_TESTING from psutil.tests import DEVNULL from psutil.tests import enum from psutil.tests import get_test_subprocess @@ -45,6 +45,7 @@ from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import mock +from psutil.tests import PYPY from psutil.tests import reap_children from psutil.tests import retry_on_failure from psutil.tests import safe_rmpath @@ -121,6 +122,8 @@ class TestSystemAPIs(unittest.TestCase): else: self.fail("subprocess not found") + @unittest.skipIf(PYPY and WINDOWS, + "get_test_subprocess() unreliable on PYPY + WINDOWS") def test_wait_procs(self): def callback(p): pids.append(p.pid) @@ -176,6 +179,8 @@ class TestSystemAPIs(unittest.TestCase): for p in gone: self.assertTrue(hasattr(p, 'returncode')) + @unittest.skipIf(PYPY and WINDOWS, + "get_test_subprocess() unreliable on PYPY + WINDOWS") def test_wait_procs_no_timeout(self): sproc1 = get_test_subprocess() sproc2 = get_test_subprocess() @@ -530,7 +535,7 @@ class TestSystemAPIs(unittest.TestCase): ls = psutil.disk_partitions(all=True) self.assertTrue(ls, msg=ls) for disk in psutil.disk_partitions(all=True): - if not WINDOWS: + if not WINDOWS and disk.mountpoint: try: os.stat(disk.mountpoint) except OSError as err: @@ -553,7 +558,7 @@ class TestSystemAPIs(unittest.TestCase): mount = find_mount_point(__file__) mounts = [x.mountpoint.lower() for x in - psutil.disk_partitions(all=True)] + psutil.disk_partitions(all=True) if x.mountpoint] self.assertIn(mount, mounts) psutil.disk_usage(mount) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index eecd7dc4..81a28807 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -9,7 +9,32 @@ Notes about unicode handling in psutil ====================================== -In psutil these are the APIs returning or dealing with a string +Starting from version 5.3.0 psutil adds unicode support, see: +https://github.com/giampaolo/psutil/issues/1040 +The notes below apply to *any* API returning a string such as +process exe(), cwd() or username(): + +* all strings are encoded by using the OS filesystem encoding + (sys.getfilesystemencoding()) which varies depending on the platform + (e.g. "UTF-8" on macOS, "mbcs" on Win) +* no API call is supposed to crash with UnicodeDecodeError +* instead, in case of badly encoded data returned by the OS, the + following error handlers are used to replace the corrupted characters in + the string: + * Python 3: sys.getfilesystemencodeerrors() (PY 3.6+) or + "surrogatescape" on POSIX and "replace" on Windows + * Python 2: "replace" +* on Python 2 all APIs return bytes (str type), never unicode +* on Python 2, you can go back to unicode by doing: + + >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") + +For a detailed explanation of how psutil handles unicode see #1040. + +Tests +===== + +List of APIs returning or dealing with a string: ('not tested' means they are not tested to deal with non-ASCII strings): * Process.cmdline() @@ -46,10 +71,6 @@ etc.) and make sure that: * psutil never crashes with UnicodeDecodeError * the returned path matches - -For a detailed explanation of how psutil handles unicode see: -- https://github.com/giampaolo/psutil/issues/1040 -- http://psutil.readthedocs.io/#unicode """ import os @@ -61,6 +82,7 @@ from psutil import BSD from psutil import MACOS from psutil import OPENBSD from psutil import POSIX +from psutil import WINDOWS from psutil._compat import PY3 from psutil._compat import u from psutil.tests import APPVEYOR @@ -194,6 +216,7 @@ class _BaseFSAPIsTests(object): if self.expect_exact_path_match(): self.assertEqual(cwd, dname) + @unittest.skipIf(PYPY and WINDOWS, "fails on PYPY + WINDOWS") def test_proc_open_files(self): p = psutil.Process() start = set(p.open_files()) @@ -261,6 +284,8 @@ class _BaseFSAPIsTests(object): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") @unittest.skipIf(not PY3, "ctypes does not support unicode on PY2") + @unittest.skipIf(PYPY and WINDOWS, + "copyload_shared_lib() unsupported on PYPY + WINDOWS") def test_memory_maps(self): # XXX: on Python 2, using ctypes.CDLL with a unicode path # opens a message box which blocks the test run. @@ -323,6 +348,7 @@ class TestNonFSAPIS(unittest.TestCase): reap_children() @unittest.skipIf(not HAS_ENVIRON, "not supported") + @unittest.skipIf(PYPY and WINDOWS, "segfaults on PYPY + WINDOWS") def test_proc_environ(self): # Note: differently from others, this test does not deal # with fs paths. On Python 2 subprocess module is broken as diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 6d4e8599..8a93743f 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -27,12 +27,14 @@ from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY from psutil.tests import mock from psutil.tests import PY3 +from psutil.tests import PYPY from psutil.tests import reap_children from psutil.tests import retry_on_failure from psutil.tests import sh from psutil.tests import unittest -if WINDOWS: + +if WINDOWS and not PYPY: with warnings.catch_warnings(): warnings.simplefilter("ignore") import win32api # requires "pip install pypiwin32" @@ -61,13 +63,18 @@ def wrap_exceptions(fun): return wrapper +@unittest.skipIf(PYPY, "pywin32 not available on PYPY") # skip whole module +class TestCase(unittest.TestCase): + pass + + # =================================================================== # System APIs # =================================================================== @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestCpuAPIs(unittest.TestCase): +class TestCpuAPIs(TestCase): @unittest.skipIf('NUMBER_OF_PROCESSORS' not in os.environ, 'NUMBER_OF_PROCESSORS env var is not available') @@ -106,7 +113,7 @@ class TestCpuAPIs(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestSystemAPIs(unittest.TestCase): +class TestSystemAPIs(TestCase): def test_nic_names(self): out = sh('ipconfig /all') @@ -230,7 +237,7 @@ class TestSystemAPIs(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestSensorsBattery(unittest.TestCase): +class TestSensorsBattery(TestCase): def test_has_battery(self): if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: @@ -291,7 +298,7 @@ class TestSensorsBattery(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestProcess(unittest.TestCase): +class TestProcess(TestCase): @classmethod def setUpClass(cls): @@ -519,7 +526,7 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestProcessWMI(unittest.TestCase): +class TestProcessWMI(TestCase): """Compare Process API results with WMI.""" @classmethod @@ -585,7 +592,7 @@ class TestProcessWMI(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestDualProcessImplementation(unittest.TestCase): +class TestDualProcessImplementation(TestCase): """ Certain APIs on Windows have 2 internal implementations, one based on documented Windows APIs, another one based @@ -668,7 +675,7 @@ class TestDualProcessImplementation(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class RemoteProcessTestCase(unittest.TestCase): +class RemoteProcessTestCase(TestCase): """Certain functions require calling ReadProcessMemory. This trivially works when called on the current process. Check that this works on other processes, especially when they @@ -764,7 +771,7 @@ class RemoteProcessTestCase(unittest.TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestServices(unittest.TestCase): +class TestServices(TestCase): def test_win_service_iter(self): valid_statuses = set([ diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index ac08c03f..fe0a73dc 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -36,6 +36,7 @@ GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" PY3 = sys.version_info[0] == 3 HERE = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.realpath(os.path.join(HERE, "..", "..")) +PYPY = '__pypy__' in sys.builtin_module_names DEPS = [ "coverage", "flake8", @@ -43,7 +44,6 @@ DEPS = [ "pdbpp", "pip", "pyperf", - "pypiwin32==219" if sys.version_info[:2] <= (3, 4) else "pypiwin32", "pyreadline", "setuptools", "wheel", @@ -56,6 +56,12 @@ if sys.version_info[:2] <= (2, 7): DEPS.append('mock') if sys.version_info[:2] <= (3, 2): DEPS.append('ipaddress') +if PYPY: + pass +elif sys.version_info[:2] <= (3, 4): + DEPS.append("pypiwin32==219") +else: + DEPS.append("pypiwin32") _cmds = {} if PY3: @@ -268,8 +274,8 @@ def upload_wheels(): def install_pip(): """Install pip""" try: - import pip # NOQA - except ImportError: + sh('%s -c "import pip"' % PYTHON) + except SystemExit: if PY3: from urllib.request import urlopen else: |