diff options
| author | Giampaolo Rodola <g.rodola@gmail.com> | 2021-10-26 23:12:24 +0200 |
|---|---|---|
| committer | Giampaolo Rodola <g.rodola@gmail.com> | 2021-10-26 23:12:24 +0200 |
| commit | 3ad8bc0aed19a5aabc2ac4a66e3fd89105b4d250 (patch) | |
| tree | 681122fa3774b71bb61fb73ae8591d675d6082b1 | |
| parent | 77586cb13e5ec4dc79294ce5fbb6253dd03926fa (diff) | |
| parent | 0e15b4890a1d84f2aa800b745fdb0642d2ee966d (diff) | |
| download | psutil-3ad8bc0aed19a5aabc2ac4a66e3fd89105b4d250.tar.gz | |
merge from master
Signed-off-by: Giampaolo Rodola <g.rodola@gmail.com>
148 files changed, 8271 insertions, 5402 deletions
diff --git a/.ci/appveyor/run_with_compiler.cmd b/.ci/appveyor/run_with_compiler.cmd index 5da547c4..7965f865 100644 --- a/.ci/appveyor/run_with_compiler.cmd +++ b/.ci/appveyor/run_with_compiler.cmd @@ -29,6 +29,7 @@ :: The CALL lines at the end of this file look redundant, but if you move them :: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y :: case, I don't know why. + @ECHO OFF SET COMMAND_TO_RUN=%* diff --git a/.ci/travis/README b/.ci/travis/README deleted file mode 100644 index d9d5f65a..00000000 --- a/.ci/travis/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains support files for Travis, a continuous integration -service which runs tests on Linux and Windows on every push. diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh deleted file mode 100755 index 1e37c39b..00000000 --- a/.ci/travis/install.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e -set -x - -uname -a -python -c "import sys; print(sys.version)" - -if [[ "$(uname -s)" == 'Darwin' ]]; then - brew update || brew update - brew outdated pyenv || brew upgrade pyenv - brew install pyenv-virtualenv - - if which pyenv > /dev/null; then - eval "$(pyenv init -)" - fi - - case "${PYVER}" in - # py26) - # pyenv install 2.6.9 - # pyenv virtualenv 2.6.9 psutil - # ;; - py27) - pyenv install 2.7.16 - pyenv virtualenv 2.7.16 psutil - ;; - py36) - pyenv install 3.6.6 - pyenv virtualenv 3.6.6 psutil - ;; - esac - pyenv rehash - pyenv activate psutil -fi - -if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]] || [[ $PYVER == 'py26' ]]; then - pip install -U ipaddress unittest2 argparse mock==1.0.1 -elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $PYVER == 'py27' ]]; then - pip install -U ipaddress mock -fi - -pip install -U coverage coveralls flake8 setuptools diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh deleted file mode 100755 index 81838633..00000000 --- a/.ci/travis/run.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -set -e -set -x - -PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` - -# setup macOS -if [[ "$(uname -s)" == 'Darwin' ]]; then - if which pyenv > /dev/null; then - eval "$(pyenv init -)" - fi - pyenv activate psutil -fi - -# install psutil -make clean -python setup.py build -python setup.py develop - -# run tests (with coverage) -if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - PSUTIL_TESTING=1 python -Wa -m coverage run psutil/tests/__main__.py -else - PSUTIL_TESTING=1 python -Wa psutil/tests/__main__.py -fi - -if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then - # run mem leaks test - PSUTIL_TESTING=1 python -Wa psutil/tests/test_memory_leaks.py - # run linter (on Linux only) - if [[ "$(uname -s)" != 'Darwin' ]]; then - python -m flake8 - fi -fi - -PSUTIL_TESTING=1 python -Wa scripts/internal/print_access_denied.py -PSUTIL_TESTING=1 python -Wa scripts/internal/print_api_speed.py diff --git a/.cirrus.yml b/.cirrus.yml deleted file mode 100644 index 41c58a93..00000000 --- a/.cirrus.yml +++ /dev/null @@ -1,31 +0,0 @@ -freebsd_12_1_py3_task: - freebsd_instance: - image: freebsd-12-1-release-amd64 - env: - CIRRUS: 1 - install_script: - - pkg install -y python3 gcc py37-pip - script: - - python3 -m pip install --user setuptools - - make clean - - make install - - make test - - make test-memleaks - - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_access_denied.py - - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_api_speed.py - -freebsd_12_1_py2_task: - freebsd_instance: - image: freebsd-12-1-release-amd64 - env: - CIRRUS: 1 - install_script: - - pkg install -y python gcc py27-pip - script: - - python2.7 -m pip install --user setuptools ipaddress mock - - make clean - - make install - - make test - - make test-memleaks - - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_access_denied.py - - PSUTIL_TESTING=1 python3 -Wa scripts/internal/print_api_speed.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..1244cd48 --- /dev/null +++ b/.flake8 @@ -0,0 +1,15 @@ +# Configuration file for flake 8. This is used by "make lint" and by the +# GIT commit hook script. +# T001 = print() statement + +[flake8] +ignore = + # line break after binary operator + W504 +per-file-ignores = + setup.py:T001 + scripts/*:T001 + scripts/internal/convert_readme.py:E501,T001 + psutil/tests/runner.py:T001 + psutil/tests/test_memleaks.py:T001 + .github/workflows/*:T001 diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index c39b2b61..03c7c77c 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,9 +1,9 @@ # These are supported funding model platforms tidelift: "pypi/psutil" -github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: giampaolo patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username +open_collective: psutil ko_fi: # Replace with a single Ko-fi username community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -custom: # Replace with a single custom sponsorship URL +custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 67a9601b..24d01efa 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -5,15 +5,17 @@ title: "[OS] title" labels: 'bug' --- -**Platform** -* { OS version } -* { psutil version: python3 -c "import psutil; print(psutil.__version__)" } -* { python version } +## Summary +* OS: { type-or-version } +* Architecture: { 64bit, 32bit, ARM, PowerPC, s390 } +* Psutil version: { pip3 show psutil } +* Python version: { python3 -V } +* Type: { core, doc, performance, scripts, tests, wheels, new-api, installation } -**Bug description** -... +## Description - -**Test results** -{ output of `python -c psutil.tests` (failures only, not full result) } +{{{ + A clear explanation of the bug, including traceback message (if any). Please read the contributing guidelines before submit: + https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md +}}} diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..39dc113f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Ask a question + url: https://groups.google.com/g/psutil + about: Use this to ask for support diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md index 7e7159f2..2f7d75a5 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -6,4 +6,14 @@ title: "[OS] title" --- -{ a clear and concise description of what the enhancment is about } +## Summary + +* OS: { type-or-version } +* Type: { core, doc, performance, scripts, tests, wheels, new-api } + +## Description + +{{{ + A clear explanation of your proposal. Please read the contributing guidelines before submit: + https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md +}}} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..e8bbb2a4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,13 @@ +## Summary + +* OS: { type-or-version } +* Bug fix: { yes/no } +* Type: { core, doc, performance, scripts, tests, wheels, new-api } +* Fixes: { comma-separated list of issues fixed by this PR, if any } + +## Description + +{{{ + A clear explanation of your bugfix or enhancement. Please read the contributing guidelines before submit: + https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md +}}} diff --git a/.github/no-response.yml b/.github/no-response.yml new file mode 100644 index 00000000..56457a28 --- /dev/null +++ b/.github/no-response.yml @@ -0,0 +1,10 @@ +# Configuration for probot-no-response: https://github.com/probot/no-response + +# Number of days of inactivity before an issue is closed for lack of response +daysUntilClose: 14 +# Label requiring a response +responseRequiredLabel: need-more-info +# Comment to post when closing an Issue for lack of response. +# Set to `false` to disable +closeComment: > + This issue has been automatically closed because there has been no response for more information from the original author. Please reach out if you have or find the answers requested so that this can be investigated further. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..0559acd2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,118 @@ +# Executed on every push by GitHub Actions. This runs CI tests and +# generates wheels (not all) on the following platforms: +# +# * Linux +# * macOS +# * Windows (commented) +# * FreeBSD +# +# To skip certain builds see: +# https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip +# +# External GH actions: +# * https://github.com/actions/checkout +# * https://github.com/actions/setup-python +# * https://github.com/actions/upload-artifact +# * https://github.com/marketplace/actions/cancel-workflow-action +# * https://github.com/vmactions/freebsd-vm + +on: [push, pull_request] +name: build +jobs: + linux-macos-win: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + # os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] + include: + - {name: Linux, python: '3.9', os: ubuntu-latest} + env: + CIBW_TEST_COMMAND: + PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py && + PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py + CIBW_TEST_EXTRAS: test + CIBW_SKIP: cp35-* pp* + + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ github.token }} + + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install cibuildwheel + run: pip install cibuildwheel + + # - name: (Windows) install Visual C++ for Python 2.7 + # if: matrix.os == 'windows-latest' + # run: | + # choco install vcpython27 -f -y + + - name: Run tests + run: cibuildwheel . + + - name: Create wheels + uses: actions/upload-artifact@v2 + with: + name: wheels + path: wheelhouse + + - name: Print hashes + if: matrix.os == 'ubuntu-latest' + run: | + make generate-manifest + python setup.py sdist + mv dist/psutil*.tar.gz wheelhouse/ + python scripts/internal/print_hashes.py wheelhouse/ + + freebsd: + runs-on: macos-latest + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ github.token }} + + - uses: actions/checkout@v2 + + - name: Run tests + id: test + uses: vmactions/freebsd-vm@v0.1.4 + with: + usesh: true + prepare: pkg install -y gcc python3 + run: | + set +e + export \ + PYTHONUNBUFFERED=1 \ + PYTHONWARNINGS=always \ + PSUTIL_DEBUG=1 + python3 -m pip install --user setuptools + python3 setup.py install + python3 psutil/tests/runner.py + python3 psutil/tests/test_memleaks.py + + linters: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: 'Run linters' + run: | + curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py + python2 get-pip.py + python2 -m pip install flake8 + python3 -m pip install flake8 + python2 -m flake8 . + python3 -m flake8 . + echo "flake8 linting OK" + find . -type f \( -iname "*.c" -o -iname "*.h" \) | xargs python3 scripts/internal/clinter.py + echo "C linting OK" diff --git a/.github/workflows/issues.py b/.github/workflows/issues.py new file mode 100644 index 00000000..ef686495 --- /dev/null +++ b/.github/workflows/issues.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Bot triggered by Github Actions every time a new issue, PR or comment +is created. Assign labels, provide replies, closes issues, etc. depending +on the situation. +""" + +import functools +import json +import os +import re +from pprint import pprint as pp + +from github import Github + + +ROOT_DIR = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..')) +SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') + + +# --- constants + + +LABELS_MAP = { + # platforms + "linux": [ + "linux", "ubuntu", "redhat", "mint", "centos", "red hat", "archlinux", + "debian", "alpine", "gentoo", "fedora", "slackware", "suse", "RHEL", + "opensuse", "manylinux", "apt ", "apt-", "rpm", "yum", "kali", + "/sys/class", "/proc/net", "/proc/disk", "/proc/smaps", + "/proc/vmstat", + ], + "windows": [ + "windows", "win32", "WinError", "WindowsError", "win10", "win7", + "win ", "mingw", "msys", "studio", "microsoft", "make.bat", + "CloseHandle", "GetLastError", "NtQuery", "DLL", "MSVC", "TCHAR", + "WCHAR", ".bat", "OpenProcess", "TerminateProcess", "appveyor", + "windows error", "NtWow64", "NTSTATUS", "Visual Studio", + ], + "macos": [ + "macos", "mac ", "osx", "os x", "mojave", "sierra", "capitan", + "yosemite", "catalina", "mojave", "big sur", "xcode", "darwin", + "dylib", "m1", + ], + "aix": ["aix"], + "cygwin": ["cygwin"], + "freebsd": ["freebsd"], + "netbsd": ["netbsd"], + "openbsd": ["openbsd"], + "sunos": ["sunos", "solaris"], + "wsl": ["wsl"], + "unix": [ + "psposix", "_psutil_posix", "waitpid", "statvfs", "/dev/tty", + "/dev/pts", + ], + "pypy": ["pypy"], + # types + "enhancement": ["enhancement"], + "memleak": ["memory leak", "leaks memory", "memleak", "mem leak"], + "api": ["idea", "proposal", "api", "feature"], + "performance": ["performance", "speedup", "speed up", "slow", "fast"], + "wheels": ["wheel", "wheels"], + "scripts": [ + "example script", "examples script", "example dir", "scripts/", + ], + # bug + "bug": [ + "fail", "can't execute", "can't install", "cannot execute", + "cannot install", "install error", "crash", "critical", + ], + # doc + "doc": [ + "doc ", "document ", "documentation", "readthedocs", "pythonhosted", + "HISTORY", "README", "dev guide", "devguide", "sphinx", "docfix", + "index.rst", + ], + # tests + "tests": [ + " test ", "tests", "travis", "coverage", "cirrus", "appveyor", + "continuous integration", "unittest", "pytest", "unit test", + ], + # critical errors + "priority-high": [ + "WinError", "WindowsError", "RuntimeError", "ZeroDivisionError", + "SystemError", "MemoryError", "core dumped", + "segfault", "segmentation fault", + ], +} + +LABELS_MAP['scripts'].extend( + [x for x in os.listdir(SCRIPTS_DIR) if x.endswith('.py')]) + +OS_LABELS = [ + "linux", "windows", "macos", "freebsd", "openbsd", "netbsd", "openbsd", + "bsd", "sunos", "unix", "wsl", "aix", "cygwin", +] + +ILLOGICAL_PAIRS = [ + ('bug', 'enhancement'), + ('doc', 'tests'), + ('scripts', 'doc'), + ('scripts', 'tests'), + ('bsd', 'freebsd'), + ('bsd', 'openbsd'), + ('bsd', 'netbsd'), +] + +# --- replies + +REPLY_MISSING_PYTHON_HEADERS = """\ +It looks like you're missing `Python.h` headers. This usually means you have \ +to install them first, then retry psutil installation. +Please read \ +[INSTALL](https://github.com/giampaolo/psutil/blob/master/INSTALL.rst) \ +instructions for your platform. \ +This is an auto-generated response based on the text you submitted. \ +If this was a mistake or you think there's a bug with psutil installation \ +process, please add a comment to reopen this issue. +""" + +# REPLY_UPDATE_CHANGELOG = """\ +# """ + + +# --- github API utils + + +def is_pr(issue): + return issue.pull_request is not None + + +def has_label(issue, label): + assigned = [x.name for x in issue.labels] + return label in assigned + + +def has_os_label(issue): + labels = set([x.name for x in issue.labels]) + for label in OS_LABELS: + if label in labels: + return True + return False + + +def get_repo(): + repo = os.environ['GITHUB_REPOSITORY'] + token = os.environ['GITHUB_TOKEN'] + return Github(token).get_repo(repo) + + +# --- event utils + + +@functools.lru_cache() +def _get_event_data(): + ret = json.load(open(os.environ["GITHUB_EVENT_PATH"])) + pp(ret) + return ret + + +def is_event_new_issue(): + data = _get_event_data() + try: + return data['action'] == 'opened' and 'issue' in data + except KeyError: + return False + + +def is_event_new_pr(): + data = _get_event_data() + try: + return data['action'] == 'opened' and 'pull_request' in data + except KeyError: + return False + + +def get_issue(): + data = _get_event_data() + try: + num = data['issue']['number'] + except KeyError: + num = data['pull_request']['number'] + return get_repo().get_issue(number=num) + + +# --- actions + + +def log(msg): + if '\n' in msg or "\r\n" in msg: + print(">>>\n%s\n<<<" % msg) + else: + print(">>> %s <<<" % msg) + + +def add_label(issue, label): + def should_add(issue, label): + if has_label(issue, label): + log("already has label %r" % (label)) + return False + + for left, right in ILLOGICAL_PAIRS: + if label == left and has_label(issue, right): + log("already has label" % (label)) + return False + + return not has_label(issue, label) + + if not should_add(issue, label): + log("should not add label %r" % label) + return + + log("add label %r" % label) + issue.add_to_labels(label) + + +def _guess_labels_from_text(issue, text): + for label, keywords in LABELS_MAP.items(): + for keyword in keywords: + if keyword.lower() in text.lower(): + yield (label, keyword) + + +def add_labels_from_text(issue, text): + for label, keyword in _guess_labels_from_text(issue, text): + add_label(issue, label) + + +def add_labels_from_new_body(issue, text): + log("start searching for template lines in new issue/PR body") + # add os label + r = re.search(r"\* OS:.*?\n", text) + log("search for 'OS: ...' line") + if r: + log("found") + add_labels_from_text(issue, r.group(0)) + else: + log("not found") + + # add bug/enhancement label + log("search for 'Bug fix: y/n' line") + r = re.search(r"\* Bug fix:.*?\n", text) + if is_pr(issue) and \ + r is not None and \ + not has_label(issue, "bug") and \ + not has_label(issue, "enhancement"): + log("found") + s = r.group(0).lower() + if 'yes' in s: + add_label(issue, 'bug') + else: + add_label(issue, 'enhancement') + else: + log("not found") + + # add type labels + log("search for 'Type: ...' line") + r = re.search(r"\* Type:.*?\n", text) + if r: + log("found") + s = r.group(0).lower() + if 'doc' in s: + add_label(issue, 'doc') + if 'performance' in s: + add_label(issue, 'performance') + if 'scripts' in s: + add_label(issue, 'scripts') + if 'tests' in s: + add_label(issue, 'tests') + if 'wheels' in s: + add_label(issue, 'wheels') + if 'new-api' in s: + add_label(issue, 'new-api') + if 'new-platform' in s: + add_label(issue, 'new-platform') + else: + log("not found") + + +# --- events + + +def on_new_issue(issue): + def has_text(text): + return text in issue.title.lower() or text in issue.body.lower() + + log("searching for missing Python.h") + if has_text("missing python.h") or \ + has_text("python.h: no such file or directory") or \ + "#include<Python.h>\n^~~~" in issue.body.replace(' ', '') or \ + "#include<Python.h>\r\n^~~~" in issue.body.replace(' ', ''): + log("found") + issue.create_comment(REPLY_MISSING_PYTHON_HEADERS) + issue.edit(state='closed') + return + + +def on_new_pr(issue): + pass + # pr = get_repo().get_pull(issue.number) + # files = [x.filename for x in list(pr.get_files())] + # if "HISTORY.rst" not in files: + # issue.create_comment(REPLY_UPDATE_CHANGELOG) + + +def main(): + issue = get_issue() + stype = "PR" if is_pr(issue) else "issue" + log("running issue bot for %s %r" % (stype, issue)) + + if is_event_new_issue(): + log("created new issue %s" % issue) + add_labels_from_text(issue, issue.title) + add_labels_from_new_body(issue, issue.body) + on_new_issue(issue) + elif is_event_new_pr(): + log("created new PR %s" % issue) + add_labels_from_text(issue, issue.title) + add_labels_from_new_body(issue, issue.body) + on_new_pr(issue) + else: + log("unhandled event") + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/issues.yml b/.github/workflows/issues.yml new file mode 100644 index 00000000..fa739eab --- /dev/null +++ b/.github/workflows/issues.yml @@ -0,0 +1,28 @@ +# Fired by Github Actions every time an issue, PR or comment is created. +name: issues +on: + issues: + types: [opened] + pull_request: + typed: [opened] + issue_comment: + types: [created] +jobs: + build: + runs-on: ubuntu-latest + steps: + # install python + - uses: actions/checkout@v2 + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + # install deps + - name: Install deps + run: python -m pip install --upgrade pip PyGithub + # run + - name: Run + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PYTHONUNBUFFERED=1 python .github/workflows/issues.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fad35121..00000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python -cache: pip -matrix: - include: - # macOS - - language: generic - os: osx - env: PYVER=py27 - - language: generic - os: osx - env: PYVER=py36 - # Linux - - python: 2.7 - - python: 3.4 - - python: 3.5 - - python: 3.6 - - python: 3.7 - - python: 3.8 - # pypy - # - python: pypy - - python: pypy3 -install: - - ./.ci/travis/install.sh -script: - - ./.ci/travis/run.sh -after_success: - - | - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - echo "sending test coverage results to coveralls.io" - coveralls - fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..481793fc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,35 @@ +Contributing to psutil project +============================== + +Issues +------ + +* The issue tracker is for reporting problems or proposing enhancements related + to the **program code**. +* Please do not open issues **asking for support**. Instead, use the forum at: + https://groups.google.com/g/psutil. +* Before submitting a new issue, **search** if there are existing issues for + the same topic. +* **Be clear** in describing what the problem is and try to be accurate in + editing the default issue **template**. There is a bot which automatically + assigns **labels** based on issue's title and body format. Labels help + keeping the issues properly organized and searchable (by OS, issue type, etc.). +* When reporting a malfunction, consider enabling + [debug mode](https://psutil.readthedocs.io/en/latest/#debug-mode) first. +* To report a **security vulnerability**, use the + [Tidelift security contact](https://tidelift.com/security). + Tidelift will coordinate the fix and the disclosure of the reported problem. + +Pull Requests +------------- + +* The PR system is for fixing bugs or make enhancements related to the + **program code**. +* If you whish to implement a new feature or add support for a new platform it's + better to **discuss it first**, either on the issue tracker, the forum or via + private email. +* In order to get acquainted with the code base and tooling, take a look at the + **[Development Guide](https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst)**. +* If you can, remember to update + [HISTORY.rst](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) + and [CREDITS](https://github.com/giampaolo/psutil/blob/master/CREDITS) file. @@ -1,25 +1,26 @@ Intro -===== +------------------------------------------------------------------------------- I would like to recognize some of the people who have been instrumental in the development of psutil. I'm sure I'm forgetting somebody (feel free to email me) but here is a short list. It's modeled after the Linux CREDITS file where the fields are: name (N), e-mail (E), website (W), country (C), description (D), -(I) issues. Issue tracker is at https://github.com/giampaolo/psutil/issues). +(I) issues. Issue tracker is at: +https://github.com/giampaolo/psutil/issues. A big thanks to all of you. - Giampaolo Author -====== +------------------------------------------------------------------------------- N: Giampaolo Rodola C: Italy E: g.rodola@gmail.com -W: http://grodola.blogspot.com/ +W: https://gmpy.dev Experts -======= +------------------------------------------------------------------------------- Github usernames of people to CC on github when in need of help. @@ -45,7 +46,7 @@ Github usernames of people to CC on github when in need of help. - wiggin15, Arnon Yaari (maintainer) Top contributors -================ +------------------------------------------------------------------------------- N: Jay Loden C: NJ, USA @@ -69,7 +70,6 @@ W: https://github.com/mrjefftang I: 340, 529, 616, 653, 654, 648, 641 N: Jeremy Whitlock -E: jcscoobyrs@gmail.com D: great help with macOS C development. I: 125, 150, 174, 206 @@ -79,7 +79,6 @@ D: OpenBSD implementation. I: 615 N: Justin Venus -E: justin.venus@gmail.com D: Solaris support I: 18 @@ -93,28 +92,122 @@ W: https://github.com/ryoon D: NetBSD implementation (co-author). I: 557 +Donations +------------------------------------------------------------------------------- + +N: aristocratos +W: https://github.com/aristocratos + +N: Daniel Widdis +C: Washington, USA +W: https://github.com/dbwiddis + +N: Rodion Stratov +C: Canada + +N: Remi Chateauneu +C: London, UK + +N: Olivier Grisel +C: Paris, France + +N: Praveen Bhamidipati +C: Bellevue, USA + +N: Willem de Groot +C: Netherlands + +N: Sigmund Vik + +N: Kahntent +C: NYC, USA + +N: Gyula Áfra +C: Budapest, Hungary + +N: Mahmut Dumlupinar + +N: Thomas Guettler +C: Germany + +N: Karthik Kumar +C: India + +N: Oche Ejembi +C: UK + +N: Russell Robinson +C: New Zealand + +N: Wompasoft +C: Texas, USA + +N: Amit Kulkarni +C: Santa Clara, USA + +N: Alexander Kaftan +C: Augsburg Germany + +N: Andrew Bays +C: Maynard, USA + +N: Carver Koella +C: Pittsburgh, USA + +N: Kristjan Võrk +C: Tallin, Estonia + +N: HTB Industries +C: Willow Springs, USA + +N: Brett Harris +C: Melbourne, Australia + +N: Peter Friedland +C: CT, USA + +N: Matthew Callow +C: Australia + +N: Marco Schrank +C: Germany + +N: Mindview LLC +C: USA + +N: Григорьев Андрей +C: Russia + +N: Heijdemann Morgan +C: Singapore + +N: Florian Bruhin +C: Winterthur, Switzerland + +N: Heijdemann Morgan +C: Singapore + +N: Morgan Heijdemann +C: Singapore + Contributors -============ +------------------------------------------------------------------------------- N: wj32 -E: wj32.64@gmail.com D: process username() and get_connections() on Windows I: 114, 115 N: Yan Raber C: Bologna, Italy -E: yanraber@gmail.com D: help on Windows development (initial version of Process.username()) N: Dave Daeschler C: USA -E: david.daeschler@gmail.com W: http://daviddaeschler.com D: some contributions to initial design/bootstrap plus occasional bug fixing I: 522, 536 N: cjgohlke -E: cjgohlke@gmail.com D: Windows 64 bit support I: 107 @@ -133,64 +226,49 @@ I: 1368, 1348 ---- N: Jeffery Kline -E: jeffery.kline@gmail.com I: 130 N: Grabriel Monnerat -E: gabrielmonnerat@gmail.com I: 146 N: Philip Roberts -E: philip.roberts@gmail.com I: 168 N: jcscoobyrs -E: jcscoobyrs@gmail.com I: 125 N: Sandro Tosi -E: sandro.tosi@gmail.com I: 200, 201 N: Andrew Colin -E: andrew.colin@gmail.com I: 248 N: Amoser -E: amoser@google.com I: 266, 267, 340 N: Matthew Grant -E: matthewgrant5@gmail.com I: 271 N: oweidner -E: oweidner@cct.lsu.edu I: 275 N: Tarek Ziade -E: ziade.tarek I: 281 N: Luca Cipriani C: Turin, Italy -E: luca.opensource@gmail.com I: 278 N: Maciej Lach, -E: maciej.lach@gmail.com I: 294 N: James Pye -E: james.pye@gmail.com I: 305, 306 N: Stanchev Emil -E: stanchev.emil I: 314 N: Kim Gräsman -E: kim.grasman@gmail.com D: ...also kindly donated some money. I: 316 @@ -199,15 +277,12 @@ C: Italy I: 318 N: Florent Xicluna -E: florent.xicluna@gmail.com I: 319 N: Michal Spondr -E: michal.spondr I: 313 N: Jean Sebastien -E: dumbboules@gmail.com I: 344 N: Rob Smith @@ -223,50 +298,39 @@ W: https://plus.google.com/116873264322260110710/posts I: 323 N: André Oriani -E: aoriani@gmail.com I: 361 N: clackwell -E: clackwell@gmail.com I: 356 N: m.malycha -E: m.malycha@gmail.com I: 351 N: John Baldwin -E: jhb@FreeBSD.org I: 370 N: Jan Beich -E: jbeich@tormail.org I: 325 N: floppymaster -E: floppymaster@gmail.com I: 380 N: Arfrever.FTA -E: Arfrever.FTA@gmail.com I: 369, 404 N: danudey -E: danudey@gmail.com I: 386 N: Adrien Fallou I: 224 N: Gisle Vanem -E: gisle.vanem@gmail.com I: 411 N: thepyr0 -E: thepyr0@gmail.com I: 414 N: John Pankov -E: john.pankov@gmail.com I: 435 N: Matt Good @@ -274,11 +338,9 @@ W: http://matt-good.net/ I: 438 N: Ulrich Klank -E: ulrich.klank@scitics.de I: 448 N: Josiah Carlson -E: josiah.carlson@gmail.com I: 451, 452 N: Raymond Hettinger @@ -291,41 +353,31 @@ M: Ken Seeho D: @cached_property decorator N: crusaderky -E: crusaderky@gmail.com I: 470, 477 -E: alex@mroja.net I: 471 N: Gautam Singh -E: gautam.singh@gmail.com I: 466 -E: lhn@hupfeldtit.dk I: 476, 479 N: Francois Charron -E: francois.charron.1@gmail.com I: 474 N: Naveed Roudsari -E: naveed.roudsari@gmail.com I: 421 N: Alexander Grothe -E: Alexander.Grothe@gmail.com I: 497 N: Szigeti Gabor Niif -E: szigeti.gabor.niif@gmail.com I: 446 N: msabramo -E: msabramo@gmail.com I: 492 N: Yaolong Huang -E: airekans@gmail.com W: http://airekans.github.io/ I: 530 @@ -335,18 +387,15 @@ I: 496 N: spacewander W: https://github.com/spacewander -E: spacewanderlzx@gmail.com I: 561, 603 N: Sylvain Mouquet -E: sylvain.mouquet@gmail.com I: 565 N: karthikrev I: 568 N: Bruno Binet -E: bruno.binet@gmail.com I: 572 N: Gabi Davar @@ -356,7 +405,6 @@ I: 578, 581, 587 N: spacewanderlzx C: Guangzhou,China -E: spacewanderlzx@gmail.com I: 555 N: Fabian Groffen @@ -373,8 +421,6 @@ C: Irvine, CA, US I: 614 N: Árni Már Jónsson -E: Reykjavik, Iceland -E: https://github.com/arnimarj I: 634 N: Bart van Kleef @@ -406,12 +452,10 @@ W: https://github.com/syohex I: 730 N: Visa Hankala -E: visa@openbsd.org I: 741 N: Sebastian-Gabriel Brestin C: Romania -E: sebastianbrestin@gmail.com I: 704 N: Timmy Konick @@ -427,12 +471,11 @@ W: https://github.com/wxwright I: 776 N: Farhan Khan -E: khanzf@gmail.com I: 823 N: Jake Omann -E: https://github.com/jomann09 -I: 816, 775 +W: https://github.com/jomann09 +I: 816, 775, 1874 N: Jeremy Humble W: https://github.com/jhumble @@ -448,7 +491,6 @@ I: 798 N: Andre Caron C: Montreal, QC, Canada -E: andre.l.caron@gmail.com W: https://github.com/AndreLouisCaron I: 880 @@ -466,7 +508,6 @@ I: 936, 1133 N: Pierre Fersing C: France -E: pierre.fersing@bleemeo.com I: 950 N: Thiago Borges Abdnur @@ -504,7 +545,7 @@ I: 1042, 1079, 1070 N: Oleksii Shevchuk W: https://github.com/alxchk -I: 1077, 1093, 1091, 1220, 1346 +I: 1077, 1093, 1091, 1220, 1346, 1904 N: Prodesire W: https://github.com/Prodesire @@ -609,9 +650,8 @@ W: https://github.com/samertm I: 1480 N: Ammar Askar -E: ammar@ammaraskar.com W: http://ammaraskar.com/ -I: 604, 1484 +I: 604, 1484, 1781 N: agnewee W: https://github.com/Agnewee @@ -660,3 +700,77 @@ I: 1648 N: Mike Hommey W: https://github.com/glandium I: 1665 + +N: Anselm Kruis +W: https://github.com/akruis +I: 1695 + +N: Michał Górny +W: https://github.com/mgorny +I: 1726 + +N: Julien Lebot +W: https://github.com/julien-lebot +I: 1768 + +N: Armin Gruner +W: https://github.com/ArminGruner +I: 1800 + +N: Chris Burger +W: https://github.com/phobozad +I: 1830 + +N: aristocratos +W: https://github.com/aristocratos +I: 1837, 1838 + +N: Vincent A. Arcila +W: https://github.com/jandrovins +I: 1620, 1727 + +N: Tim Schlueter +W: https://github.com/modelrockettier +I: 1822 + +N: marxin +W: https://github.com/marxin +I: 1851 + +N: guille +W: https://github.com/guille +I: 1913 + +N: David Knaack +W: https://github.com/davidkna +I: 1921 + +N: Nikita Radchenko +W: https://github.com/nradchenko +I: 1940 + +N: MaWe2019 +W: https://github.com/MaWe2019 +I: 1953 + +N: Dmitry Gorbunov +C: Russia +E: gorbunov.dmitry.1999@gmail.com +W: https://gorbunov-dmitry.github.io +D: fix typos in documentation + +N: Pablo Baeyens +W: https://github.com/mx-psi +I: 1598 + +N: Xuehai Pan +W: https://github.com/XuehaiPan +I: 1948 + +N: Saeed Rasooli +W: https://github.com/ilius +I: 1996 + +N: PetrPospisil +W: https://github.com/PetrPospisil +I: 1980 diff --git a/HISTORY.rst b/HISTORY.rst index 52b46f7f..5531f9d1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,22 +1,180 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.7.0 (unreleased) -================== +5.8.1 (IN DEVELOPMENT) +====================== XXXX-XX-XX **Enhancements** -- 1637_: [SunOS] Add partial support for old SunOS 5.10 Update 0 to 3. +- 1851_: [Linux] cpu_freq() is slow on systems with many CPUs. Read current + frequency values for all CPUs from /proc/cpuinfo instead of opening many + files in /sys fs. (patch by marxin) +- 1992_: NoSuchProcess message now specifies if the PID has been reused. +- 1992_: error classes (NoSuchProcess, AccessDenied, etc.) now have a better + formatted and separated `__repr__` and `__str__` implementations. +- 1996_: add support for MidnightBSD. (patch by Saeed Rasooli) +- 1999_: [Linux] disk_partitions(): convert "/dev/root" device (an alias used + on some Linux distros) to real root device path. +- 2005_: PSUTIL_DEBUG mode now prints file name and line number of the debug + messages coming from C extension modules. + +**Bug fixes** + +- 1456_: [macOS] psutil.cpu_freq()'s min and max are set to 0 if can't be + determined (instead of crashing). +- 1512_: [macOS] sometimes Process.connections() will crash with EOPNOTSUPP + for one connection; this is now ignored. +- 1598_: [Windows] psutil.disk_partitions() only returns mountpoints on drives + where it first finds one +- 1874_: [Solaris] swap output error due to incorrect range. +- 1892_: [macOS] psutil.cpu_freq() broken on Apple M1. +- 1901_: [macOS] different functions, especially process' open_files() and + connections() methods, could randomly raise AccessDenied because the internal + buffer of `proc_pidinfo(PROC_PIDLISTFDS)` syscall was not big enough. We now + dynamically increase the buffer size until it's big enough instead of giving + up and raising AccessDenied, which was a fallback to avoid crashing. +- 1904_: [Windows] OpenProcess fails with ERROR_SUCCESS due to GetLastError() + called after sprintf(). (patch by alxchk) +- 1913_: [Linux] wait_procs seemingly ignoring timeout, TimeoutExpired thrown +- 1919_: [Linux] sensors_battery() can raise TypeError on PureOS. +- 1921_: [Windows] psutil.swap_memory() shows committed memory instead of swap +- 1940_: [Linux] psutil does not handle ENAMETOOLONG when accessing process + file descriptors in procfs. (patch by Nikita Radchenko) +- 1948_: Process' memoize_when_activated decorator was not thread-safe. (patch + by Xuehai Pan) +- 1953_: [Windows] disk_partitions() crashes due to insufficient buffer len. + (patch by MaWe2019) +- 1965_: [Windows] fix "Fatal Python error: deallocating None" when calling + psutil.users() multiple times. +- 1980_: [Windows] 32bit / WOW64 processes fails to read process name longer + than 128 characters resulting in AccessDenied. This is now fixed. (patch + by PetrPospisil) +- 1991_: process_iter() can raise TypeError if invoked from multiple threads + (not thread-safe). + +5.8.0 +===== + +2020-12-19 + +**Enhancements** + +- 1863_: `disk_partitions()` exposes 2 extra fields: `maxfile` and `maxpath`, + which are the maximum file name and path name length. +- 1872_: [Windows] added support for PyPy 2.7. +- 1879_: provide pre-compiled wheels for Linux and macOS (yey!). +- 1880_: get rid of Travis and Cirrus CI services (they are no longer free). + CI testing is now done by GitHub Actions on Linux, macOS and FreeBSD (yes). + AppVeyor is still being used for Windows CI. + +**Bug fixes** + +- 1708_: [Linux] get rid of sensors_temperatures() duplicates. (patch by Tim + Schlueter). +- 1839_: [Windows] always raise AccessDenied when failing to query 64 processes + from 32 bit ones (NtWoW64 APIs). +- 1866_: [Windows] process exe(), cmdline(), environ() may raise "invalid + access to memory location" on Python 3.9. +- 1874_: [Solaris] wrong swap output given when encrypted column is present. +- 1875_: [Windows] process username() may raise ERROR_NONE_MAPPED if the SID + has no corresponding account name. In this case AccessDenied is now raised. +- 1877_: [Windows] OpenProcess may fail with ERROR_SUCCESS. Turn it into + AccessDenied or NoSuchProcess depending on whether the PID is alive. +- 1886_: [macOS] EIO error may be raised on cmdline() and environment(). Now + it gets translated into AccessDenied. +- 1891_: [macOS] get rid of deprecated getpagesize(). + +5.7.3 +===== + +2020-10-23 + +**Enhancements** + +- 809_: [FreeBSD] add support for `Process.rlimit()`. +- 893_: [BSD] add support for `Process.environ()` (patch by Armin Gruner) +- 1830_: [UNIX] `net_if_stats()`'s `isup` also checks whether the NIC is + running (meaning Wi-Fi or ethernet cable is connected). (patch by Chris Burger) +- 1837_: [Linux] improved battery detection and charge "secsleft" calculation + (patch by aristocratos) + +**Bug fixes** + +- 1620_: [Linux] cpu_count(logical=False) result is incorrect on systems with + more than one CPU socket. (patch by Vincent A. Arcila) +- 1738_: [macOS] Process.exe() may raise FileNotFoundError if process is still + alive but the exe file which launched it got deleted. +- 1791_: [macOS] fix missing include for getpagesize(). +- 1823_: [Windows] Process.open_files() may cause a segfault due to a NULL + pointer. +- 1838_: [Linux] sensors_battery(): if `percent` can be determined but not + the remaining values, still return a result instead of None. + (patch by aristocratos) + +5.7.2 +===== + +2020-07-15 + +**Bug fixes** + +- wheels for 2.7 were inadvertently deleted. + +5.7.1 +===== + +2020-07-15 + +**Enhancements** + +- 1729_: parallel tests on UNIX (make test-parallel). They're twice as fast! +- 1741_: "make build/install" is now run in parallel and it's about 15% faster + on UNIX. +- 1747_: `Process.wait()` on POSIX returns an enum, showing the negative signal + which was used to terminate the process:: + >>> import psutil + >>> p = psutil.Process(9891) + >>> p.terminate() + >>> p.wait() + <Negsignal.SIGTERM: -15> +- 1747_: `Process.wait()` return value is cached so that the exit code can be + retrieved on then next call. +- 1747_: Process provides more info about the process on str() and repr() + (status and exit code):: + >>> proc + psutil.Process(pid=12739, name='python3', status='terminated', + exitcode=<Negsigs.SIGTERM: -15>, started='15:08:20') +- 1757_: memory leak tests are now stable. +- 1768_: [Windows] added support for Windows Nano Server. (contributed by + Julien Lebot) + +**Bug fixes** + +- 1726_: [Linux] cpu_freq() parsing should use spaces instead of tabs on ia64. + (patch by Michał Górny) +- 1760_: [Linux] Process.rlimit() does not handle long long type properly. +- 1766_: [macOS] NoSuchProcess may be raised instead of ZombieProcess. +- 1781_: fix signature of callback function for getloadavg(). (patch by + Ammar Askar) + +5.7.0 +===== + +2020-02-18 + +**Enhancements** + +- 1637_: [SunOS] add partial support for old SunOS 5.10 Update 0 to 3. - 1648_: [Linux] sensors_temperatures() looks into an additional /sys/device/ directory for additional data. (patch by Javad Karabi) - 1652_: [Windows] dropped support for Windows XP and Windows Server 2003. Minimum supported Windows version now is Windows Vista. -- 1667_: added process_iter(new_only=True) parameter. - 1671_: [FreeBSD] add CI testing/service for FreeBSD (Cirrus CI). - 1677_: [Windows] process exe() will succeed for all process PIDs (instead of raising AccessDenied). - 1679_: [Windows] net_connections() and Process.connections() are 10% faster. +- 1682_: [PyPy] added CI / test integration for PyPy via Travis. - 1686_: [Windows] added support for PyPy on Windows. - 1693_: [Windows] boot_time(), Process.create_time() and users()'s login time now have 1 micro second precision (before the precision was of 1 second). @@ -41,6 +199,8 @@ XXXX-XX-XX - 1674_: [SunOS] disk_partitions() may raise OSError. - 1684_: [Linux] disk_io_counters() may raise ValueError on systems not having /proc/diskstats. +- 1695_: [Linux] could not compile on kernels <= 2.6.13 due to + PSUTIL_HAVE_IOPRIO not being defined. (patch by Anselm Kruis) 5.6.7 ===== @@ -138,7 +298,7 @@ XXXX-XX-XX average calculation, including on Windows (emulated). (patch by Ammar Askar) - 1404_: [Linux] cpu_count(logical=False) uses a second method (read from `/sys/devices/system/cpu/cpu[0-9]/topology/core_id`) in order to determine - the number of physical CPUs in case /proc/cpuinfo does not provide this info. + the number of CPU cores in case /proc/cpuinfo does not provide this info. - 1458_: provide coloured test output. Also show failures on KeyboardInterrupt. - 1464_: various docfixes (always point to python3 doc, fix links, etc.). - 1476_: [Windows] it is now possible to set process high I/O priority @@ -384,7 +544,7 @@ XXXX-XX-XX - 694_: [SunOS] cmdline() could be truncated at the 15th character when reading it from /proc. An extra effort is made by reading it from process address space first. (patch by Georg Sauthoff) -- 771_: [Windows] cpu_count() (both logical and physical) return a wrong +- 771_: [Windows] cpu_count() (both logical and cores) return a wrong (smaller) number on systems using process groups (> 64 cores). - 771_: [Windows] cpu_times(percpu=True) return fewer CPUs on systems using process groups (> 64 cores). @@ -1188,8 +1348,8 @@ XXXX-XX-XX - 593_: [FreeBSD] Process().memory_maps() segfaults. - 606_: Process.parent() may swallow NoSuchProcess exceptions. - 611_: [SunOS] net_io_counters has send and received swapped -- 614_: [Linux]: cpu_count(logical=False) return the number of physical CPUs - instead of physical cores. +- 614_: [Linux]: cpu_count(logical=False) return the number of sockets instead + of cores. - 618_: [SunOS] swap tests fail on Solaris when run as normal user - 628_: [Linux] Process.name() truncates process name in case it contains spaces or parentheses. @@ -1306,7 +1466,7 @@ XXXX-XX-XX **Enhancements** - 424_: [Windows] installer for Python 3.X 64 bit. -- 427_: number of logical and physical CPUs (psutil.cpu_count()). +- 427_: number of logical CPUs and physical cores (psutil.cpu_count()). - 447_: psutil.wait_procs() timeout parameter is now optional. - 452_: make Process instances hashable and usable with set()s. - 453_: tests on Python < 2.7 require unittest2 module. diff --git a/INSTALL.rst b/INSTALL.rst index c3b9e91c..c7efea5e 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,140 +1,119 @@ -Install pip -=========== - -pip is the easiest way to install psutil. It is shipped by default with Python -2.7.9+ and 3.4+. For other Python versions you can install it manually. -On Linux or via wget:: - - wget https://bootstrap.pypa.io/get-pip.py -O - | python - -On macOS or via curl:: - - python < <(curl -s https://bootstrap.pypa.io/get-pip.py) - -On Windows, `download pip <https://pip.pypa.io/en/latest/installing/>`__, open -cmd.exe and install it:: - - C:\Python27\python.exe get-pip.py - -Permission issues (UNIX) -======================== +Install psutil +============== -The commands below assume you're running as root. -If you aren't or you bump into permission errors you can either install psutil -for your user only:: +Linux, Windows, macOS (wheels) +------------------------------ - pip3 install --user psutil +Pre-compiled wheels are distributed for these platforms, so you won't have to +install a C compiler. All you have to do is:: -...or prepend ``sudo`` and install it globally, e.g.:: + pip install psutil - sudo pip3 install psutil +If wheels are not available for your platform or architecture, or you whish to +build & install psutil from sources, keep reading. -Linux -===== +Linux (build) +------------- Ubuntu / Debian:: sudo apt-get install gcc python3-dev - pip3 install psutil + pip install --no-binary :all: psutil RedHat / CentOS:: sudo yum install gcc python3-devel - pip3 install psutil + pip install --no-binary :all: psutil -If you're on Python 2 use ``python-dev`` instead. - -macOS -===== - -Install `Xcode <https://developer.apple.com/downloads/?name=Xcode>`__ then run:: - - pip3 install psutil - -Windows -======= - -Open a cmd.exe shell and run:: - - python3 -m pip install psutil +Windows (build) +--------------- -This assumes "python" is in your PATH. If not, specify the full python.exe -path. +In order to install psutil from sources on Windows you need Visual Studio +(MinGW is not supported). +Here's a couple of guides describing how to do it: `link <https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/>`__ +and `link <https://cpython-core-tutorial.readthedocs.io/en/latest/build_cpython_windows.html>`__. +Once VS is installed do:: -In order to compile psutil from sources you'll need **Visual Studio** (Mingw32 -is not supported). -This `blog post <https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/>`__ -provides numerous info on how to properly set up VS. The needed VS versions are: + pip install --no-binary :all: psutil -* Python 2.6, 2.7: `VS-2008 <http://www.microsoft.com/en-us/download/details.aspx?id=44266>`__ -* Python 3.4: `VS-2010 <http://www.visualstudio.com/downloads/download-visual-studio-vs#d-2010-express>`__ -* Python 3.5+: `VS-2015 <http://www.visualstudio.com/en-au/news/vs2015-preview-vs>`__ +macOS (build) +------------- -Compiling 64 bit versions of Python 2.6 and 2.7 with VS 2008 requires -`Windows SDK and .NET Framework 3.5 SP1 <https://www.microsoft.com/en-us/download/details.aspx?id=3138>`__. -Once installed run `vcvars64.bat` -(see `here <http://stackoverflow.com/questions/11072521/>`__). -Once VS is setup open a cmd.exe shell, cd into psutil directory and run:: +Install `Xcode <https://developer.apple.com/downloads/?name=Xcode>`__ then run:: - python3 setup.py build - python3 setup.py install + pip install --no-binary :all: psutil FreeBSD -======= +------- :: pkg install python3 gcc - python -m pip3 install psutil + python3 -m pip install psutil OpenBSD -======= +------- :: export PKG_PATH=http://ftp.eu.openbsd.org/pub/OpenBSD/`uname -r`/packages/`uname -m`/ pkg_add -v python gcc - python3 -m pip install psutil + pip install psutil NetBSD -====== +------ :: export PKG_PATH="ftp.netbsd.org/pub/pkgsrc/packages/NetBSD/`uname -m`/`uname -r`/All" pkg_add -v pkgin pkgin install python3 gcc - python3 -m pip install psutil + pip install psutil -Solaris -======= +Sun Solaris +----------- -If ``cc`` compiler is not installed create a symlink to ``gcc``:: +If ``cc`` compiler is not installed create a symbolic link to ``gcc``:: sudo ln -s /usr/bin/gcc /usr/local/bin/cc Install:: pkg install gcc - python3 -m pip install psutil + pip install psutil -Install from sources -==================== +Troubleshooting +=============== -:: +Install pip +----------- - git clone https://github.com/giampaolo/psutil.git - cd psutil - python3 setup.py install +Pip is shipped by default with Python 2.7.9+ and 3.4+. +If you don't have pip you can install with wget:: -Testing installation -==================== + wget https://bootstrap.pypa.io/get-pip.py -O - | python3 -:: +...or with curl:: + + python3 < <(curl -s https://bootstrap.pypa.io/get-pip.py) + +On Windows, `download pip <https://pip.pypa.io/en/latest/installing/>`__, open +cmd.exe and install it with:: + + py get-pip.py + +"pip not found" +--------------- + +Sometimes pip is installed but it's not available in your ``PATH`` +("pip command not found" or similar). Try this:: + + python3 -m pip install psutil - python3 -m psutil.tests +Permission errors (UNIX) +------------------------ -Dev Guide -========= +If you want to install psutil system-wide and you bump into permission errors +either run as root user or prepend ``sudo``:: -See: `dev guide <https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst>`__. + sudo pip install psutil diff --git a/MANIFEST.in b/MANIFEST.in index 038d4baa..e9c20d81 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,7 @@ include .coveragerc +include .flake8 include .gitignore +include CONTRIBUTING.md include CREDITS include HISTORY.rst include INSTALL.rst @@ -46,6 +48,8 @@ include psutil/arch/aix/ifaddrs.h include psutil/arch/aix/net_connections.c include psutil/arch/aix/net_connections.h include psutil/arch/aix/net_kernel_structs.h +include psutil/arch/freebsd/cpu.c +include psutil/arch/freebsd/cpu.h include psutil/arch/freebsd/proc_socks.c include psutil/arch/freebsd/proc_socks.h include psutil/arch/freebsd/specific.c @@ -58,6 +62,8 @@ include psutil/arch/netbsd/specific.c include psutil/arch/netbsd/specific.h include psutil/arch/openbsd/specific.c include psutil/arch/openbsd/specific.h +include psutil/arch/osx/cpu.c +include psutil/arch/osx/cpu.h include psutil/arch/osx/process_info.c include psutil/arch/osx/process_info.h include psutil/arch/solaris/environ.c @@ -94,13 +100,14 @@ include psutil/tests/test_bsd.py include psutil/tests/test_connections.py include psutil/tests/test_contracts.py include psutil/tests/test_linux.py -include psutil/tests/test_memory_leaks.py +include psutil/tests/test_memleaks.py include psutil/tests/test_misc.py include psutil/tests/test_osx.py include psutil/tests/test_posix.py include psutil/tests/test_process.py include psutil/tests/test_sunos.py include psutil/tests/test_system.py +include psutil/tests/test_testutils.py include psutil/tests/test_unicode.py include psutil/tests/test_windows.py include scripts/battery.py @@ -109,20 +116,26 @@ include scripts/disk_usage.py include scripts/fans.py include scripts/free.py include scripts/ifconfig.py -include scripts/internal/.git-pre-commit include scripts/internal/README include scripts/internal/bench_oneshot.py include scripts/internal/bench_oneshot_2.py include scripts/internal/check_broken_links.py -include scripts/internal/download_exes.py +include scripts/internal/clinter.py +include scripts/internal/convert_readme.py +include scripts/internal/download_wheels_appveyor.py +include scripts/internal/download_wheels_github.py include scripts/internal/fix_flake8.py include scripts/internal/generate_manifest.py +include scripts/internal/git_pre_commit.py include scripts/internal/print_access_denied.py include scripts/internal/print_announce.py include scripts/internal/print_api_speed.py +include scripts/internal/print_downloads.py +include scripts/internal/print_hashes.py include scripts/internal/print_timeline.py +include scripts/internal/print_wheels.py include scripts/internal/purge_installation.py -include scripts/internal/scriptutils.py +include scripts/internal/tidelift.py include scripts/internal/winmake.py include scripts/iotop.py include scripts/killall.py @@ -141,4 +154,3 @@ include scripts/top.py include scripts/who.py include scripts/winservices.py include setup.py -include tox.ini @@ -2,30 +2,44 @@ # To use a specific Python version run: "make install PYTHON=python3.3" # You can set the variables below from the command line. +# Configurable. PYTHON = python3 -TSCRIPT = psutil/tests/__main__.py ARGS = -# List of nice-to-have dev libs. +TSCRIPT = psutil/tests/runner.py + +# Internal. DEPS = \ argparse \ check-manifest \ + concurrencytest \ coverage \ flake8 \ - futures \ - ipaddress \ - mock==1.0.1 \ + flake8-print \ pyperf \ + pypinfo \ requests \ setuptools \ - sphinx==2.3.1 \ + sphinx_rtd_theme \ twine \ - unittest2 \ virtualenv \ wheel - +PY2_DEPS = \ + futures \ + ipaddress \ + mock==1.0.1 \ + unittest2 +DEPS += `$(PYTHON) -c \ + "import sys; print('$(PY2_DEPS)' if sys.version_info[0] == 2 else '')"` +# "python3 setup.py build" can be parallelized on Python >= 3.6. +BUILD_OPTS = `$(PYTHON) -c \ + "import sys, os; \ + py36 = sys.version_info[:2] >= (3, 6); \ + cpus = os.cpu_count() or 1 if py36 else 1; \ + print('--parallel %s' % cpus if cpus > 1 else '')"` # In not in a virtualenv, add --user options for install commands. -INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` -TEST_PREFIX = PYTHONWARNINGS=all PSUTIL_TESTING=1 PSUTIL_DEBUG=1 +INSTALL_OPTS = `$(PYTHON) -c \ + "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` +TEST_PREFIX = PYTHONWARNINGS=always PSUTIL_DEBUG=1 all: test @@ -34,7 +48,7 @@ all: test # =================================================================== clean: ## Remove all build files. - rm -rf `find . -type d -name __pycache__ \ + @rm -rfv `find . -type d -name __pycache__ \ -o -type f -name \*.bak \ -o -type f -name \*.orig \ -o -type f -name \*.pyc \ @@ -44,13 +58,12 @@ clean: ## Remove all build files. -o -type f -name \*.so \ -o -type f -name \*.~ \ -o -type f -name \*\$testfn` - rm -rf \ + @rm -rfv \ *.core \ *.egg-info \ - *\$testfn* \ + *\@psutil-* \ .coverage \ .failed-tests.txt \ - .tox \ build/ \ dist/ \ docs/_build/ \ @@ -58,26 +71,25 @@ clean: ## Remove all build files. _: -build: _ ## Compile without installing. - # make sure setuptools is installed (needed for 'develop' / edit mode) - $(PYTHON) -c "import setuptools" - PYTHONWARNINGS=all $(PYTHON) setup.py build - @# copies compiled *.so files in ./psutil directory in order to allow - @# "import psutil" when using the interactive interpreter from within - @# this directory. - PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i - $(PYTHON) -c "import psutil" # make sure it actually worked +build: _ ## Compile (in parallel) without installing. + @# "build_ext -i" copies compiled *.so files in ./psutil directory in order + @# to allow "import psutil" when using the interactive interpreter from + @# within this directory. + PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i $(BUILD_OPTS) install: ## Install this package as current user in "edit" mode. + # make sure setuptools is installed (needed for 'develop' / edit mode) + $(PYTHON) -c "import setuptools" ${MAKE} build PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS) + $(PYTHON) -c "import psutil" # make sure it actually worked uninstall: ## Uninstall this package via pip. cd ..; $(PYTHON) -m pip uninstall -y -v psutil || true $(PYTHON) scripts/internal/purge_installation.py install-pip: ## Install pip (no-op if already installed). - $(PYTHON) -c \ + @$(PYTHON) -c \ "import sys, ssl, os, pkgutil, tempfile, atexit; \ sys.exit(0) if pkgutil.find_loader('pip') else None; \ pyexc = 'from urllib.request import urlopen' if sys.version_info[0] == 3 else 'from urllib2 import urlopen'; \ @@ -91,7 +103,7 @@ install-pip: ## Install pip (no-op if already installed). f.write(data); \ f.flush(); \ print('downloaded %s' % f.name); \ - code = os.system('%s %s --user' % (sys.executable, f.name)); \ + code = os.system('%s %s --user --upgrade' % (sys.executable, f.name)); \ f.close(); \ sys.exit(code);" @@ -106,55 +118,63 @@ setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). # =================================================================== test: ## Run all tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) + +test-parallel: ## Run all tests in parallel. + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) --parallel test-process: ## Run process-related API tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_process.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_process.py test-system: ## Run system-related API tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_system.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_system.py test-misc: ## Run miscellaneous tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_misc.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_misc.py + +test-testutils: ## Run test utils tests. + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_testutils.py test-unicode: ## Test APIs dealing with strings. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_unicode.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_unicode.py test-contracts: ## APIs sanity tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_contracts.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_contracts.py test-connections: ## Test net_connections() and Process.connections(). - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_connections.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_connections.py test-posix: ## POSIX specific tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_posix.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_posix.py test-platform: ## Run specific platform tests only. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) psutil/tests/test_memory_leaks.py + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) psutil/tests/test_memleaks.py test-by-name: ## e.g. make test-by-name ARGS=psutil.tests.test_system.TestSystemAPIs - ${MAKE} install - @$(TEST_PREFIX) $(PYTHON) -m unittest -v $(ARGS) + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) test-failed: ## Re-run tests which failed on last run - ${MAKE} install - $(TEST_PREFIX) $(PYTHON) -c "import psutil.tests.runner as r; r.run(last_failed=True)" + ${MAKE} build + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) $(ARGS) --last-failed test-coverage: ## Run test coverage. - ${MAKE} install + ${MAKE} build # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov $(TEST_PREFIX) $(PYTHON) -m coverage run $(TSCRIPT) @@ -167,43 +187,58 @@ test-coverage: ## Run test coverage. # Linters # =================================================================== -flake8: ## flake8 linter. - @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 +lint-py: ## Run Python (flake8) linter. + @git ls-files '*.py' | xargs $(PYTHON) -m flake8 --config=.flake8 + +lint-c: ## Run C linter. + @git ls-files '*.c' '*.h' | xargs $(PYTHON) scripts/internal/clinter.py -fix-flake8: ## Attempt to automaticall fix some flake8 issues. +lint: ## Run Python (flake8) and C linters. + ${MAKE} lint-py + ${MAKE} lint-c + +fix-lint: ## Attempt to automatically fix some Python lint issues. @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 --exit-zero | $(PYTHON) scripts/internal/fix_flake8.py # =================================================================== # GIT # =================================================================== -git-tag-release: ## Git-tag a new release. - git tag -a release-`python -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD` - git push --follow-tags - install-git-hooks: ## Install GIT pre-commit hook. - ln -sf ../../scripts/internal/.git-pre-commit .git/hooks/pre-commit + ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit chmod +x .git/hooks/pre-commit # =================================================================== +# Wheels +# =================================================================== + +download-wheels-github: ## Download latest wheels hosted on github. + $(PYTHON) scripts/internal/download_wheels_github.py --tokenfile=~/.github.token + +download-wheels-appveyor: ## Download latest wheels hosted on appveyor. + $(PYTHON) scripts/internal/download_wheels_appveyor.py + +print-wheels: ## Print downloaded wheels + $(PYTHON) scripts/internal/print_wheels.py + +# =================================================================== # Distribution # =================================================================== +git-tag-release: ## Git-tag a new release. + git tag -a release-`python -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD` + git push --follow-tags + sdist: ## Create tar.gz source distribution. ${MAKE} generate-manifest $(PYTHON) setup.py sdist - -wheel: ## Generate wheel. - $(PYTHON) setup.py bdist_wheel - -win-download-wheels: ## Download wheels hosted on appveyor. - $(TEST_PREFIX) $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil + $(PYTHON) -m twine check dist/*.tar.gz upload-src: ## Upload source tarball on https://pypi.org/project/psutil/ ${MAKE} sdist - $(PYTHON) setup.py sdist upload + $(PYTHON) -m twine upload dist/*.tar.gz -upload-win-wheels: ## Upload wheels in dist/* directory on PyPI. +upload-wheels: ## Upload wheels in dist/* directory on PyPI. $(PYTHON) -m twine upload dist/*.whl # --- others @@ -216,13 +251,20 @@ check-sdist: ## Create source distribution and checks its sanity (MANIFEST) build/venv/bin/python -m pip install -v --isolated --quiet dist/*.tar.gz build/venv/bin/python -c "import os; os.chdir('build/venv'); import psutil" +tidelift-relnotes: ## upload release notes from HISTORY + $(PYTHON) scripts/internal/tidelift.py + pre-release: ## Check if we're ready to produce a new release. ${MAKE} check-sdist ${MAKE} install ${MAKE} generate-manifest git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain - ${MAKE} win-download-wheels ${MAKE} sdist + ${MAKE} download-wheels-github + ${MAKE} download-wheels-appveyor + ${MAKE} print-hashes + ${MAKE} print-wheels + $(PYTHON) -m twine check dist/* $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ @@ -230,12 +272,12 @@ pre-release: ## Check if we're ready to produce a new release. assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" - $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff --quiet && git diff --cached --quiet', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else 0 ;" release: ## Create a release (down/uploads tar.gz, wheels, git tag release). - ${MAKE} pre-release + $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff --quiet && git diff --cached --quiet', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else 0 ;" $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PyPI ${MAKE} git-tag-release + ${MAKE} tidelift-relnotes check-manifest: ## Inspect MANIFEST.in file. $(PYTHON) -m check_manifest -v $(ARGS) @@ -254,13 +296,19 @@ print-timeline: ## Print releases' timeline. @$(PYTHON) scripts/internal/print_timeline.py print-access-denied: ## Print AD exceptions - ${MAKE} install + ${MAKE} build @$(TEST_PREFIX) $(PYTHON) scripts/internal/print_access_denied.py print-api-speed: ## Benchmark all API calls - ${MAKE} install + ${MAKE} build @$(TEST_PREFIX) $(PYTHON) scripts/internal/print_api_speed.py $(ARGS) +print-downloads: ## Print PYPI download statistics + $(PYTHON) scripts/internal/print_downloads.py + +print-hashes: ## Prints hashes of files in dist/ directory + $(PYTHON) scripts/internal/print_hashes.py dist/ + # =================================================================== # Misc # =================================================================== @@ -269,11 +317,11 @@ grep-todos: ## Look for TODOs in the source files. git grep -EIn "TODO|FIXME|XXX" bench-oneshot: ## Benchmarks for oneshot() ctx manager (see #799). - ${MAKE} install + ${MAKE} build $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot.py bench-oneshot-2: ## Same as above but using perf module (supposed to be more precise) - ${MAKE} install + ${MAKE} build $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot_2.py check-broken-links: ## Look for broken links in source files. @@ -1,6 +1,6 @@ | |downloads| |stars| |forks| |contributors| |coverage| |quality| | |version| |py-versions| |packages| |license| -| |travis| |appveyor| |cirrus| |doc| |twitter| |tidelift| +| |github-actions| |appveyor| |doc| |twitter| |tidelift| .. |downloads| image:: https://img.shields.io/pypi/dm/psutil.svg :target: https://pepy.tech/project/psutil @@ -22,24 +22,20 @@ :target: https://www.codacy.com/app/g-rodola/psutil?utm_source=github.com&utm_medium=referral&utm_content=giampaolo/psutil&utm_campaign=Badge_Grade :alt: Code quality -.. |travis| image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux,%20OSX,%20PyPy - :target: https://travis-ci.org/giampaolo/psutil - :alt: Linux tests (Travis) +.. |github-actions| image:: https://img.shields.io/github/workflow/status/giampaolo/psutil/CI?label=Linux%2C%20macOS%2C%20FreeBSD + :target: https://github.com/giampaolo/psutil/actions?query=workflow%3Abuild + :alt: Linux, macOS, Windows tests .. |appveyor| image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=Windows :target: https://ci.appveyor.com/project/giampaolo/psutil :alt: Windows tests (Appveyor) -.. |cirrus| image:: https://img.shields.io/cirrus/github/giampaolo/psutil?label=FreeBSD - :target: https://cirrus-ci.com/github/giampaolo/psutil-cirrus-ci - :alt: FreeBSD tests (Cirrus-Ci) - -.. |coverage| image:: https://img.shields.io/coveralls/github/giampaolo/psutil.svg?label=test%20coverage +.. |coverage| image:: https://coveralls.io/repos/github/giampaolo/psutil/badge.svg?branch=master :target: https://coveralls.io/github/giampaolo/psutil?branch=master :alt: Test coverage (coverall.io) .. |doc| image:: https://readthedocs.org/projects/psutil/badge/?version=latest - :target: http://psutil.readthedocs.io/en/latest/?badge=latest + :target: https://psutil.readthedocs.io/en/latest/ :alt: Documentation Status .. |version| image:: https://img.shields.io/pypi/v/psutil.svg?label=pypi @@ -47,7 +43,6 @@ :alt: Latest version .. |py-versions| image:: https://img.shields.io/pypi/pyversions/psutil.svg - :target: https://pypi.org/project/psutil :alt: Supported Python versions .. |packages| image:: https://repology.org/badge/tiny-repos/python:psutil.svg @@ -68,18 +63,21 @@ ----- -Quick links -=========== - -- `Home page <https://github.com/giampaolo/psutil>`_ -- `Install <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`_ -- `Documentation <http://psutil.readthedocs.io>`_ -- `Download <https://pypi.org/project/psutil/#files>`_ -- `Forum <http://groups.google.com/group/psutil/topics>`_ -- `StackOverflow <https://stackoverflow.com/questions/tagged/psutil>`_ -- `Blog <http://grodola.blogspot.com/search/label/psutil>`_ -- `Development guide <https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst>`_ -- `What's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst>`_ +.. raw:: html + + <div align="center"> + <a href="https://github.com/giampaolo/psutil"><img src="https://github.com/giampaolo/psutil/raw/master/docs/_static/psutil-logo.png" /></a> + <br /> + <br /> + <a href="https://github.com/giampaolo/psutil"><b>Home</b></a> + <a href="https://github.com/giampaolo/psutil/blob/master/INSTALL.rst"><b>Install</b></a> + <a href="https://psutil.readthedocs.io/"><b>Documentation</b></a> + <a href="https://pypi.org/project/psutil/#files"><b>Download</b></a> + <a href="https://groups.google.com/g/psutil"><b>Forum</b></a> + <a href="https://gmpy.dev/tags/psutil"><b>Blog</b></a> + <a href="#funding"><b>Funding</b></a> + <a href="https://github.com/giampaolo/psutil/blob/master/HISTORY.rst"><b>What's new</b></a> + </div> Summary ======= @@ -100,81 +98,58 @@ psutil currently supports the following platforms: - **Sun Solaris** - **AIX** -...both **32-bit** and **64-bit** architectures. Supported Python versions are **2.6**, **2.7** and **3.4+**. `PyPy3 <http://pypy.org/>`__ is also known to work. - -psutil for enterprise -===================== - -.. |tideliftlogo| image:: https://nedbatchelder.com/pix/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png - :width: 150 - :alt: Tidelift - :target: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme - -.. list-table:: - :widths: 10 150 - - * - |tideliftlogo| - - The maintainer of psutil and thousands of other packages are working - with Tidelift to deliver commercial support and maintenance for the open - source dependencies you use to build your applications. Save time, - reduce risk, and improve code health, while paying the maintainers of - the exact dependencies you use. - `Learn more <https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=enterprise&utm_term=repo>`__. +Supported Python versions are **2.6**, **2.7**, **3.4+** and +`PyPy <http://pypy.org/>`__. - By subscribing to Tidelift you will help me (`Giampaolo Rodola`_) support - psutil future development. Alternatively consider making a small - `donation`_. +Funding +======= -Security +While psutil is free software and will always be, the project would benefit +immensely from some funding. +Keeping up with bug reports and maintenance has become hardly sustainable for +me alone in terms of time. +If you're a company that's making significant use of psutil you can consider +becoming a sponsor via `GitHub Sponsors <https://github.com/sponsors/giampaolo>`__, +`Open Collective <https://opencollective.com/psutil>`__ or +`PayPal <https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8>`__ +and have your logo displayed in here and psutil `doc <https://psutil.readthedocs.io>`__. + +Sponsors ======== -To report a security vulnerability, please use the `Tidelift security -contact`_. Tidelift will coordinate the fix and disclosure. +.. raw:: html -Example applications -==================== + <div> + <a href="https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme"> + <img width="185" src="https://github.com/giampaolo/psutil/raw/master/docs/_static/tidelift-logo.svg" /> + </a> +   + <a href="https://sansec.io/"> + <img src="https://sansec.io/assets/images/logo.svg" /> + </a> + </div> + <sup><a href="https://github.com/sponsors/giampaolo">add your logo</a></sup> -+------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ -| .. image:: https://github.com/giampaolo/psutil/blob/master/docs/_static/procinfo-small.png | .. image:: https://github.com/giampaolo/psutil/blob/master/docs/_static/top-small.png | -| :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/procinfo.png | :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/top.png | -+------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ -| .. image:: https://github.com/giampaolo/psutil/blob/master/docs/_static/procsmem-small.png | .. image:: https://github.com/giampaolo/psutil/blob/master/docs/_static/pmap-small.png | -| :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/procsmem.png | :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/pmap.png | -+------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ +Supporters +========== -Also see `scripts directory <https://github.com/giampaolo/psutil/tree/master/scripts>`__ -and `doc recipes <http://psutil.readthedocs.io/#recipes/>`__. +.. raw:: html -Projects using psutil -===================== + <div> + <a href="https://github.com/dbwiddis"><img height="40" width="40" title="Daniel Widdis" src="https://avatars1.githubusercontent.com/u/9291703?s=88&v=4" /></a> + <a href="https://github.com/aristocratos"><img height="40" width="40" title="aristocratos" src="https://avatars3.githubusercontent.com/u/59659483?s=96&v=4" /></a> + <a href="https://github.com/cybersecgeek"><img height="40" width="40" title="cybersecgeek" src="https://avatars.githubusercontent.com/u/12847926?v=4" /></a> + <a href="https://github.com/scoutapm-sponsorships"><img height="40" width="40" title="scoutapm-sponsorships" src="https://avatars.githubusercontent.com/u/71095532?v=4" /></a> + <a href="https://opencollective.com/chenyoo-hao"><img height="40" width="40" title="Chenyoo Hao" src="https://images.opencollective.com/chenyoo-hao/avatar/40.png" /></a> + <a href="https://opencollective.com/alexey-vazhnov"><img height="40" width="40" title="Alexey Vazhnov" src="https://images.opencollective.com/alexey-vazhnov/daed334/avatar/40.png" /></a> + </div> + <sup><a href="https://github.com/sponsors/giampaolo">add your avatar</a></sup> -psutil has roughly the following monthly downloads: -.. image:: https://img.shields.io/pypi/dm/psutil.svg - :target: https://pepy.tech/project/psutil - :alt: Downloads - -There are over -`10.000 open source projects <https://libraries.io/pypi/psutil/dependent_repositories?page=1>`__ -on github which depend from psutil. -Here's some I find particularly interesting: - -- https://github.com/google/grr -- https://github.com/facebook/osquery/ -- https://github.com/nicolargo/glances -- https://github.com/Jahaja/psdash -- https://github.com/ajenti/ajenti -- https://github.com/home-assistant/home-assistant/ - - -Portings -======== - -- Go: https://github.com/shirou/gopsutil -- C: https://github.com/hamon-in/cpslib -- Rust: https://github.com/borntyping/rust-psutil -- Nim: https://github.com/johnscillieri/psutil-nim +Contributing +============ +See `contributing guidelines <https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md>`__. Example usages ============== @@ -243,8 +218,8 @@ Disks .. code-block:: python >>> psutil.disk_partitions() - [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid'), - sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext, opts='rw')] + [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid', maxfile=255, maxpath=4096), + sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext, opts='rw', maxfile=255, maxpath=4096)] >>> >>> psutil.disk_usage('/') sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) @@ -262,7 +237,7 @@ Network {'eth0': netio(bytes_sent=485291293, bytes_recv=6004858642, packets_sent=3251564, packets_recv=4787798, errin=0, errout=0, dropin=0, dropout=0), 'lo': netio(bytes_sent=2838627, bytes_recv=2838627, packets_sent=30567, packets_recv=30567, errin=0, errout=0, dropin=0, dropout=0)} >>> - >>> psutil.net_connections() + >>> psutil.net_connections(kind='tcp') [sconn(fd=115, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED', pid=1254), sconn(fd=117, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING', pid=2987), ...] @@ -320,18 +295,20 @@ Process management >>> import psutil >>> psutil.pids() - [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, 1216, 1220, 1221, 1243, 1244, - 1301, 1601, 2237, 2355, 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, 4263, 4282, - 4306, 4311, 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, 4395, 4408, 4433, 4443, 4445, 4446, - 5167, 5234, 5235, 5252, 5318, 5424, 5644, 6987, 7054, 7055, 7071] + [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, + 1216, 1220, 1221, 1243, 1244, 1301, 1601, 2237, 2355, 2637, 2774, 3932, + 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, 4263, 4282, 4306, 4311, + 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, 4395, 4408, 4433, + 4443, 4445, 4446, 5167, 5234, 5235, 5252, 5318, 5424, 5644, 6987, 7054, + 7055, 7071] >>> >>> p = psutil.Process(7055) >>> p - psutil.Process(pid=7055, name='python', started='09:04:44') + psutil.Process(pid=7055, name='python3', status='running', started='09:04:44') >>> p.name() - 'python' + 'python3' >>> p.exe() - '/usr/bin/python' + '/usr/bin/python3' >>> p.cwd() '/home/giampaolo' >>> p.cmdline() @@ -342,15 +319,15 @@ Process management >>> p.ppid() 7054 >>> p.children(recursive=True) - [psutil.Process(pid=29835, name='python2.7', started='11:45:38'), - psutil.Process(pid=29836, name='python2.7', started='11:43:39')] + [psutil.Process(pid=29835, name='python3', status='sleeping', started='11:45:38'), + psutil.Process(pid=29836, name='python3', status='waking', started='11:43:39')] >>> >>> p.parent() - psutil.Process(pid=4699, name='bash', started='09:06:44') + psutil.Process(pid=4699, name='bash', status='sleeping', started='09:06:44') >>> p.parents() [psutil.Process(pid=4699, name='bash', started='09:06:44'), - psutil.Process(pid=4689, name='gnome-terminal-server', started='0:06:44'), - psutil.Process(pid=1, name='systemd', started='05:56:55')] + psutil.Process(pid=4689, name='gnome-terminal-server', status='sleeping', started='0:06:44'), + psutil.Process(pid=1, name='systemd', status='sleeping', started='05:56:55')] >>> >>> p.status() 'running' @@ -396,7 +373,7 @@ Process management [popenfile(path='/home/giampaolo/monit.py', fd=3, position=0, mode='r', flags=32768), popenfile(path='/var/log/monit.log', fd=4, position=235542, mode='a', flags=33793)] >>> - >>> p.connections() + >>> p.connections(kind='tcp') [pconn(fd=115, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'), pconn(fd=117, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING')] >>> @@ -438,7 +415,7 @@ Process management >>> p.terminate() >>> p.kill() >>> p.wait(timeout=3) - 0 + <Exitcode.EX_OK: 0> >>> >>> psutil.test() USER PID %CPU %MEM VSZ RSS TTY START TIME COMMAND @@ -456,7 +433,7 @@ Further process APIs .. code-block:: python >>> import psutil - >>> for proc in psutil.process_iter(attrs=['pid', 'name']): + >>> for proc in psutil.process_iter(['pid', 'name']): ... print(proc.info) ... {'pid': 1, 'name': 'systemd'} @@ -474,23 +451,6 @@ Further process APIs >>> gone, alive = psutil.wait_procs(procs_list, timeout=3, callback=on_terminate) >>> -Popen wrapper: - -.. code-block:: python - - >>> import psutil - >>> from subprocess import PIPE - >>> p = psutil.Popen(["/usr/bin/python", "-c", "print('hello')"], stdout=PIPE) - >>> p.name() - 'python' - >>> p.username() - 'giampaolo' - >>> p.communicate() - ('hello\n', None) - >>> p.wait(timeout=2) - 0 - >>> - Windows services ---------------- @@ -513,9 +473,22 @@ Windows services 'status': 'stopped', 'username': 'NT AUTHORITY\\LocalService'} +Projects using psutil +===================== + +Here's some I find particularly interesting: + +- https://github.com/google/grr +- https://github.com/facebook/osquery/ +- https://github.com/nicolargo/glances +- https://github.com/Jahaja/psdash +- https://github.com/ajenti/ajenti +- https://github.com/home-assistant/home-assistant/ -.. _`Giampaolo Rodola`: http://grodola.blogspot.com/p/about.html -.. _`donation`: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 -.. _Tidelift security contact: https://tidelift.com/security -.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme +Portings +======== +- Go: https://github.com/shirou/gopsutil +- C: https://github.com/hamon-in/cpslib +- Rust: https://github.com/rust-psutil/rust-psutil +- Nim: https://github.com/johnscillieri/psutil-nim diff --git a/appveyor.yml b/appveyor.yml index b38cbf1b..59c47f0a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,13 +1,22 @@ -# Build: 2 (bump this up by 1 to force an appveyor run) +# Build: 3 (bump this up by 1 to force an appveyor run) os: Visual Studio 2015 - +# avoid 2 builds when pushing on PRs +skip_branch_with_pr: true +# avoid build on new GIT tag +skip_tags: true +matrix: + # stop build on first failure + fast_finish: true environment: global: # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script intepreter # See: http://stackoverflow.com/a/13751649/163740 WITH_COMPILER: "cmd /E:ON /V:ON /C .\\.ci\\appveyor\\run_with_compiler.cmd" + PYTHONWARNINGS: always + PYTHONUNBUFFERED: 1 + PSUTIL_DEBUG: 1 matrix: # 32 bits @@ -15,10 +24,6 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python35" - PYTHON_VERSION: "3.5.x" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Python36" PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "32" @@ -31,16 +36,17 @@ environment: PYTHON_VERSION: "3.8.x" PYTHON_ARCH: "32" + - PYTHON: "C:\\Python39" + PYTHON_VERSION: "3.9.x" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + PYTHON_ARCH: "32" + # 64 bits - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.x" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" @@ -53,6 +59,11 @@ environment: PYTHON_VERSION: "3.8.x" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python39-x64" + PYTHON_VERSION: "3.9.x" + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + PYTHON_ARCH: "64" + init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" @@ -69,9 +80,11 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py test-memleaks" after_test: - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py wheel" + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/print_hashes.py dist" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/print_access_denied.py" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/print_api_speed.py" @@ -85,7 +98,7 @@ cache: # - might want to upload the content of dist/*.whl to a public wheelhouse skip_commits: - message: skip-ci + message: skip-appveyor # run build only if one of the following files is modified on commit only_commits: diff --git a/docs/DEVGUIDE.rst b/docs/DEVGUIDE.rst index e07d977e..cb9545bc 100644 --- a/docs/DEVGUIDE.rst +++ b/docs/DEVGUIDE.rst @@ -1,178 +1,136 @@ +psutil development guide +======================== + Build, setup and running tests -=============================== +.............................. + +psutil makes extensive use of C extension modules, meaning a C compiler is +required, see +`install instructions <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`__. -Make sure to `install <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`__ -a C compiler first, then: +Once you have a compiler installed: .. code-block:: bash - git clone git@github.com:giampaolo/psutil.git - make setup-dev-env - make test + git clone git@github.com:giampaolo/psutil.git + make setup-dev-env # install useful dev libs (flake8, coverage, ...) + make build + make install + make test -- bear in mind that ``make``(see `Makefile`_) is the designated tool to run - tests, build, install etc. and that it is also available on Windows (see +- ``make`` (see `Makefile`_) is the designated tool to run tests, build, install + try new features you're developing, etc. This also includes Windows (see `make.bat`_ ). -- do not use ``sudo``; ``make install`` installs psutil as a limited user in - "edit" mode; also ``make setup-dev-env`` installs deps as a limited user. -- use `make help` to see the list of available commands. - -Coding style -============ - -- python code strictly follows `PEP-8`_ styling guides and this is enforced by - a commit GIT hook installed via ``make install-git-hooks``. -- C code follows `PEP-7`_ styling guides. - -Makefile -======== - -Some useful make commands: + Some useful commands are: .. code-block:: bash - make install # install - make setup-dev-env # install useful dev libs (flake8, unittest2, etc.) - make test # run unit tests - make test-memleaks # run memory leak tests - make test-coverage # run test coverage - make flake8 # run PEP8 linter + make test-parallel # faster + make test-memleaks + make test-coverage + make lint # Python (PEP8) and C linters + make uninstall + make help -There are some differences between ``make`` on UNIX and Windows. -For instance, to run a specific Python version. On UNIX: +- if you're working on a new feature and you whish to compile & test it "on the + fly" from a test script, this is a quick & dirty way to do it: .. code-block:: bash - make test PYTHON=python3.5 - -On Windows: + make test TSCRIPT=test_script.py # on UNIX + make test test_script.py # on Windows -.. code-block:: bat +- do not use ``sudo``. ``make install`` installs psutil as a limited user in + "edit" mode, meaning you can edit psutil code on the fly while you develop. - make -p C:\python35\python.exe test - -If you want to modify psutil and run a script on the fly which uses it do -(on UNIX): +- if you want to target a specific Python version: .. code-block:: bash - make test TSCRIPT=foo.py - -On Windows: + make test PYTHON=python3.5 # UNIX + make test -p C:\python35\python.exe # Windows -.. code-block:: bat - - make test foo.py +Coding style +------------ -Adding a new feature -==================== +- python code strictly follows `PEP-8`_ styling guides and this is enforced by + a commit GIT hook installed via ``make install-git-hooks`` which will reject + commits if code is not PEP-8 complieant. +- C code should follow `PEP-7`_ styling guides. -Usually the files involved when adding a new functionality are: +Code organization +----------------- .. code-block:: bash - psutil/__init__.py # main psutil namespace - psutil/_ps{platform}.py # python platform wrapper - psutil/_psutil_{platform}.c # C platform extension - psutil/_psutil_{platform}.h # C header file + psutil/__init__.py # main psutil namespace ("import psutil") + psutil/_ps{platform}.py # platform-specific python wrapper + psutil/_psutil_{platform}.c # platform-specific C extension psutil/tests/test_process|system.py # main test suite - psutil/tests/test_{platform}.py # platform specific test suite + psutil/tests/test_{platform}.py # platform-specific test suite -Typical process occurring when adding a new functionality (API): +Adding a new API +---------------- -- define the new function in `psutil/__init__.py`_. +Typically, this is what you do: + +- define the new API in `psutil/__init__.py`_. - write the platform specific implementation in ``psutil/_ps{platform}.py`` (e.g. `psutil/_pslinux.py`_). -- if the change requires C, write the C implementation in +- if the change requires C code, write the C implementation in ``psutil/_psutil_{platform}.c`` (e.g. `psutil/_psutil_linux.c`_). - write a generic test in `psutil/tests/test_system.py`_ or `psutil/tests/test_process.py`_. -- if possible, write a platform specific test in +- if possible, write a platform-specific test in ``psutil/tests/test_{platform}.py`` (e.g. `psutil/tests/test_linux.py`_). - This usually means testing the return value of the new feature against + This usually means testing the return value of the new API against a system CLI tool. -- update doc in ``doc/index.py``. -- update ``HISTORY.rst``. +- update the doc in ``doc/index.py``. +- update `HISTORY.rst`_ and `CREDITS`_ files. - make a pull request. Make a pull request -=================== +------------------- - fork psutil (go to https://github.com/giampaolo/psutil and click on "fork") -- git clone your fork locally: ``git clone git@github.com:YOUR-USERNAME/psutil.git``) -- create your feature branch:``git checkout -b new-feature`` +- git clone the fork locally: ``git clone git@github.com:YOUR-USERNAME/psutil.git`` +- create a branch:``git checkout -b new-feature`` - commit your changes: ``git commit -am 'add some feature'`` -- push to the branch: ``git push origin new-feature`` -- create a new pull request by via github web interface +- push the branch: ``git push origin new-feature`` +- create a new PR via the GitHub web interface and sign-off your work (see + `CONTRIBUTING.md`_ guidelines) Continuous integration -====================== - -All of the services listed below are automatically run on ``git push``. +---------------------- -Unit tests ----------- +Unit tests are automatically run on every ``git push`` on **Linux**, **macOS**, +**Windows** and **FreeBSD** by using: -Tests are automatically run for every GIT push on **Linux**, **macOS** and -**Windows** by using: - -- `Travis`_ (Linux, macOS) +- `Github Actions`_ (Linux, macOS, Windows) - `Appveyor`_ (Windows) -Test files controlling these are `.travis.yml`_ and `appveyor.yml`_. -Both services run psutil test suite against all supported python version -(2.6 - 3.6). -Two icons in the home page (README) always show the build status: - -.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux,%20OSX,%20PyPy - :target: https://travis-ci.org/giampaolo/psutil - :alt: Linux, macOS and PyPy3 tests (Travis) +.. image:: https://img.shields.io/github/workflow/status/giampaolo/psutil/CI?label=Linux%2C%20macOS%2C%20FreeBSD + :target: https://github.com/giampaolo/psutil/actions?query=workflow%3ACI .. image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=Windows :target: https://ci.appveyor.com/project/giampaolo/psutil - :alt: Windows tests (Appveyor) - -.. image:: https://img.shields.io/cirrus/github/giampaolo/psutil?label=FreeBSD - :target: https://cirrus-ci.com/github/giampaolo/psutil-cirrus-ci - :alt: FreeBSD tests (Cirrus-CI) - -BSD, AIX and Solaris are currently tested manually. - -Test coverage -------------- - -Test coverage is provided by `coveralls.io`_ and it is controlled via -`.travis.yml`_. -An icon in the home page (README) always shows the last coverage percentage: -.. image:: https://coveralls.io/repos/giampaolo/psutil/badge.svg?branch=master&service=github - :target: https://coveralls.io/github/giampaolo/psutil?branch=master - :alt: Test coverage (coverall.io) +OpenBSD, NetBSD, AIX and Solaris does not have continuos test integration. Documentation -============= +------------- - doc source code is written in a single file: `/docs/index.rst`_. -- it uses `RsT syntax`_ - and it's built with `sphinx`_. - doc can be built with ``make setup-dev-env; cd docs; make html``. -- public doc is hosted on http://psutil.readthedocs.io/ - -Releasing a new version -======================= - -These are notes for myself (Giampaolo): - -- ``make release`` -- post announce (``make print-announce``) on psutil and python-announce mailing - lists, twitter, g+, blog. - +- public doc is hosted at https://psutil.readthedocs.io -.. _`.travis.yml`: https://github.com/giampaolo/psutil/blob/master/.travis.ym -.. _`appveyor.yml`: https://github.com/giampaolo/psutil/blob/master/appveyor.ym +.. _`appveyor.yml`: https://github.com/giampaolo/psutil/blob/master/appveyor.yml .. _`Appveyor`: https://ci.appveyor.com/project/giampaolo/psuti .. _`coveralls.io`: https://coveralls.io/github/giampaolo/psuti +.. _`CREDITS`: https://github.com/giampaolo/psutil/blob/master/CREDITS +.. _`CONTRIBUTING.md`: https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md .. _`doc/index.rst`: https://github.com/giampaolo/psutil/blob/master/doc/index.rst +.. _`Github Actions`: https://github.com/giampaolo/psutil/actions .. _`HISTORY.rst`: https://github.com/giampaolo/psutil/blob/master/HISTORY.rst .. _`make.bat`: https://github.com/giampaolo/psutil/blob/master/make.bat .. _`Makefile`: https://github.com/giampaolo/psutil/blob/master/Makefile @@ -186,4 +144,3 @@ These are notes for myself (Giampaolo): .. _`psutil/tests/test_system.py`: https://github.com/giampaolo/psutil/blob/master/psutil/tests/test_system.py .. _`RsT syntax`: http://docutils.sourceforge.net/docs/user/rst/quickref.htm .. _`sphinx`: http://sphinx-doc.org -.. _`Travis`: https://travis-ci.org/giampaolo/psuti diff --git a/docs/DEVNOTES b/docs/DEVNOTES index abd2e368..1748bfda 100644 --- a/docs/DEVNOTES +++ b/docs/DEVNOTES @@ -5,51 +5,11 @@ A collection of ideas and notes about stuff to implement in future versions. "#NNN" occurrences refer to bug tracker issues at: https://github.com/giampaolo/psutil/issues -PLATFORMS -========= - -- #355: Android (with patch) -- #82: Cygwin (PR at #998) -- #276: GNU/Hurd -- #693: Windows Nano -- #1251: Windows bash -- DragonFlyBSD -- HP-UX - FEATURES ======== -- set process name/title - -- #1115: users() idle time. - -- #1102: Process.is64bit(). - -- #371: sensors_temperatures() at least for macOS. - -- #669: Windows / net_if_addrs(): return broadcast addr. - -- #550: CPU info (frequency, architecture, threads per core, cores per socket, - sockets, ...) - -- #772: extended net_io_counters() metrics. - -- #900: wheels for macOS and Linux. - -- #922: extended net_io_stats() info. - -- #914: extended platform specific process info. - -- #898: wifi stats - -- #893: (BSD) process environ - -- #809: (BSD) per-process resource limits (rlimit()). - - (UNIX) process root (different from cwd) -- #782: (UNIX) process num of signals received. - - (Linux) locked files via /proc/locks: https://www.centos.org/docs/5/html/5.2/Deployment_Guide/s2-proc-locks.html @@ -58,16 +18,14 @@ FEATURES Linux: yes Others: ? -- Process.threads(): thread names; patch for macOS available at: - https://code.google.com/p/plcrashreporter/issues/detail?id=65 - Sample code: - https://github.com/janmojzis/pstree/blob/master/proc_kvm.c - - Asynchronous psutil.Popen (see http://bugs.python.org/issue1191964) - (Windows) fall back on using WMIC for Process methods returning AccessDenied -- #613: thread names. +- #613: thread names; patch for macOS available at: + https://code.google.com/p/plcrashreporter/issues/detail?id=65 + Sample code: + https://github.com/janmojzis/pstree/blob/master/proc_kvm.c - scripts/taskmgr-gui.py (using tk). @@ -139,29 +97,18 @@ FEATURES NoSuchProcess and AccessDenied? Not that we need it, but currently we cannot raise a TimeoutExpired exception with a specific error string. -- process_iter() might grow an "attrs" parameter similar to Process.as_dict() - invoke the necessary methods and include the results into a "cache" - attribute attached to the returned Process instances so that one can avoid - catching NSP and AccessDenied: - for p in process_iter(attrs=['cpu_percent']): - print(p.cache['cpu_percent']) - This also leads questions as whether we should introduce a sorting order. - - round Process.memory_percent() result? -- #550: number of threads per core. - -- cpu_percent() and cpu_times_percent() use global vars so are not thread safe. - BUGFIXES ======== - #600: windows / open_files(): support network file handles. -REJECTED -======== +REJECTED IDEAS +============== - #550: threads per core +- #1667: process_iter(new_only=True) INCONSISTENCIES =============== diff --git a/docs/Makefile b/docs/Makefile index c7f4723a..860a2b0e 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -15,6 +15,10 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +DEPS = \ + sphinx \ + sphinx_rtd_theme + .PHONY: help help: @echo "Please use \`make <target>' where <target> is one of" @@ -224,3 +228,7 @@ dummy: $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy @echo @echo "Build finished. Dummy builder generates no files." + +.PHONY: setup-dev-env +setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). + $(PYTHON) -m pip install --user --upgrade --trusted-host files.pythonhosted.org $(DEPS) diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index c5c201e4..e88f5307 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -4,7 +4,7 @@ } .rst-content dl:not(.docutils) { - margin: 0px 0px 0px 0px; + margin: 0px 0px 0px 0px !important; } .data dd { diff --git a/docs/_static/psutil-logo.png b/docs/_static/psutil-logo.png Binary files differnew file mode 100644 index 00000000..e1c67233 --- /dev/null +++ b/docs/_static/psutil-logo.png diff --git a/docs/_static/tidelift-logo.svg b/docs/_static/tidelift-logo.svg new file mode 100644 index 00000000..af12d684 --- /dev/null +++ b/docs/_static/tidelift-logo.svg @@ -0,0 +1,33 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Generator: Adobe Illustrator 21.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) --> +<svg version="1.1" id="Artwork" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" + viewBox="0 0 190.1 33" style="enable-background:new 0 0 190.1 33;" xml:space="preserve"> +<style type="text/css"> + .st0{fill:#4B5168;} + .st1{fill:#F6914D;} +</style> +<g> + <path class="st0" d="M33.4,27.7V5.3c0-2.3,0-2.3,2.4-2.3c2.4,0,2.4,0,2.4,2.3v22.4c0,2.3,0,2.3-2.4,2.3 + C33.4,29.9,33.4,29.9,33.4,27.7z"/> + <path class="st0" d="M45,26.4V6.6c0-3.6,0-3.6,3.6-3.6h5.8c7.8,0,12.5,3.9,13,10.2c0.2,2.2,0.2,3.4,0,5.5 + c-0.5,6.3-5.3,11.2-13,11.2h-5.8C45,29.9,45,29.9,45,26.4z M54.3,25.4c5.3,0,8-3,8.3-7.1c0.1-1.8,0.1-2.8,0-4.6 + c-0.3-4.2-3-6.1-8.3-6.1h-4.5v17.8H54.3z"/> + <path class="st0" d="M73.8,26.4V6.6c0-3.6,0-3.6,3.6-3.6h13.5c2.3,0,2.3,0,2.3,2.2c0,2.2,0,2.2-2.3,2.2H78.6v6.9h11 + c2.2,0,2.2,0,2.2,2.1c0,2.1,0,2.1-2.2,2.1h-11v6.9h12.3c2.3,0,2.3,0,2.3,2.2c0,2.3,0,2.3-2.3,2.3H77.4 + C73.8,29.9,73.8,29.9,73.8,26.4z"/> + <path class="st0" d="M100,26.4v-21c0-2.3,0-2.3,2.4-2.3c2.4,0,2.4,0,2.4,2.3v20.2h11.9c2.4,0,2.4,0,2.4,2.2c0,2.2,0,2.2-2.4,2.2 + h-13.1C100,29.9,100,29.9,100,26.4z"/> + <path class="st0" d="M125.8,27.7V5.3c0-2.3,0-2.3,2.4-2.3c2.4,0,2.4,0,2.4,2.3v22.4c0,2.3,0,2.3-2.4,2.3 + C125.8,29.9,125.8,29.9,125.8,27.7z"/> + <path class="st0" d="M137.4,27.7V6.6c0-3.6,0-3.6,3.6-3.6h13.5c2.3,0,2.3,0,2.3,2.2c0,2.2,0,2.2-2.3,2.2h-12.2v7.2h11.3 + c2.3,0,2.3,0,2.3,2.2c0,2.2,0,2.2-2.3,2.2h-11.3v8.6c0,2.3,0,2.3-2.4,2.3S137.4,29.9,137.4,27.7z"/> + <path class="st0" d="M24.2,3.1H5.5c-2.4,0-2.4,0-2.4,2.2c0,2.2,0,2.2,2.4,2.2h7v4.7v3.2l4.8-3.7v-1.1V7.5h7c2.4,0,2.4,0,2.4-2.2 + C26.6,3.1,26.6,3.1,24.2,3.1z"/> + <path class="st1" d="M12.5,20v7.6c0,2.3,0,2.3,2.4,2.3c2.4,0,2.4,0,2.4-2.3V16.3L12.5,20z"/> + <g> + <path class="st0" d="M165.9,3.1h18.7c2.4,0,2.4,0,2.4,2.2c0,2.2,0,2.2-2.4,2.2h-7v4.7v3.2l-4.8-3.7v-1.1V7.5h-7 + c-2.4,0-2.4,0-2.4-2.2C163.5,3.1,163.5,3.1,165.9,3.1z"/> + <path class="st1" d="M177.6,20v7.6c0,2.3,0,2.3-2.4,2.3c-2.4,0-2.4,0-2.4-2.3V16.3L177.6,20z"/> + </g> +</g> +</svg> diff --git a/docs/conf.py b/docs/conf.py index b056d20f..f4a32b98 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -267,21 +267,21 @@ htmlhelp_basename = '%s-doc' % PROJECT_NAME # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples diff --git a/docs/index.rst b/docs/index.rst index 82b4498f..4b3c1272 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,9 +10,10 @@ Quick links - `Home page <https://github.com/giampaolo/psutil>`__ - `Install <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`_ -- `Blog <http://grodola.blogspot.com/search/label/psutil>`__ - `Forum <http://groups.google.com/group/psutil/topics>`__ - `Download <https://pypi.org/project/psutil/#files>`__ +- `Blog <https://gmpy.dev/tags/psutil>`__ +- `Contributing <https://github.com/giampaolo/psutil/blob/master/CONTRIBUTING.md>`__ - `Development guide <https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst>`_ - `What's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst>`__ @@ -42,40 +43,59 @@ Supported Python versions are **2.6**, **2.7** and **3.4+**. The psutil documentation you're reading is distributed as a single HTML page. +Funding +======= -Professional support --------------------- - -.. image:: https://nedbatchelder.com/pix/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png - :width: 80px - :align: left - -Professional support for psutil is available as part of the `Tidelift Subscription`_. -Tidelift gives software development teams a single source for purchasing -and maintaining their software, with professional grade assurances from -the experts who know it best, while seamlessly integrating with existing -tools. -By subscribing you will help me (`Giampaolo Rodola`_) support psutil -future development. Alternatively consider making a small `donation`_. -To report a security vulnerability, please use the `Tidelift security -contact`_. Tidelift will coordinate the fix and disclosure. - -Install -------- - -Linux Ubuntu / Debian:: +While psutil is free software and will always be, the project would benefit +immensely from some funding. +Keeping up with bug reports and maintenance has become hardly sustainable for +me alone in terms of time. +If you're a company that's making significant use of psutil you can consider +becoming a sponsor via `GitHub <https://github.com/sponsors/giampaolo>`__, +`Open Collective <https://opencollective.com/psutil>`__ or +`PayPal <https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8>`__ +and have your logo displayed in here and psutil `doc <https://psutil.readthedocs.io>`__. + +Sponsors +-------- + +.. raw:: html + + <div> + <a href="https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme"> + <img width="185" src="https://github.com/giampaolo/psutil/raw/master/docs/_static/tidelift-logo.svg" /> + </a> +   + <a href="https://sansec.io/"> + <img src="https://sansec.io/assets/images/logo.svg" /> + </a> + </div> + + <br /> + <sup><a href="https://github.com/sponsors/giampaolo">add your logo</a></sup> + +Supporters +---------- - sudo apt-get install gcc python3-dev - sudo pip3 install psutil +.. raw:: html -Linux Redhat:: + <div> + <a href="https://github.com/dbwiddis"><img height="40" width="40" title="Daniel Widdis" src="https://avatars1.githubusercontent.com/u/9291703?s=88&v=4" /></a> + <a href="https://github.com/aristocratos"><img height="40" width="40" title="aristocratos" src="https://avatars3.githubusercontent.com/u/59659483?s=96&v=4" /></a> + <a href="https://github.com/cybersecgeek"><img height="40" width="40" title="cybersecgeek" src="https://avatars.githubusercontent.com/u/12847926?v=4" /></a> + <a href="https://github.com/scoutapm-sponsorships"><img height="40" width="40" title="scoutapm-sponsorships" src="https://avatars.githubusercontent.com/u/71095532?v=4" /></a> + <a href="https://opencollective.com/chenyoo-hao"><img height="40" width="40" title="Chenyoo Hao" src="https://images.opencollective.com/chenyoo-hao/avatar/40.png" /></a> + <a href="https://opencollective.com/alexey-vazhnov"><img height="40" width="40" title="Alexey Vazhnov" src="https://images.opencollective.com/alexey-vazhnov/daed334/avatar/40.png" /></a> + </div> + <br /> + <sup><a href="https://github.com/sponsors/giampaolo">add your avatar</a></sup> - sudo yum install gcc python3-devel - sudo pip3 install psutil +Install +======= -Windows:: +On Linux, Windows, macOS:: - pip3 install psutil + pip install psutil For other platforms see more detailed `install <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`_ @@ -131,6 +151,12 @@ CPU .. versionchanged:: 4.1.0 added *interrupt* and *dpc* fields on Windows. + .. warning:: + CPU times are always supposed to increase over time, or at least remain + the same, and that's because time cannot go backwards. + Surprisingly sometimes this might not be the case (at least on Windows + and Linux), see `#1210 <https://github.com/giampaolo/psutil/issues/1210#issuecomment-363046156>`__. + .. function:: cpu_percent(interval=None, percpu=False) Return a float representing the current system-wide CPU utilization as a @@ -186,13 +212,13 @@ CPU Return the number of logical CPUs in the system (same as `os.cpu_count`_ in Python 3.4) or ``None`` if undetermined. - *logical* cores means the number of physical cores multiplied by the number + "logical CPUs" means the number of physical cores multiplied by the number of threads that can run on each core (this is known as Hyper Threading). - If *logical* is ``False`` return the number of physical cores only (Hyper - Thread CPUs are excluded) or ``None`` if undetermined. + If *logical* is ``False`` return the number of physical cores only, or + ``None`` if undetermined. On OpenBSD and NetBSD ``psutil.cpu_count(logical=False)`` always return ``None``. - Example on a system having 2 physical hyper-thread CPU cores: + Example on a system having 2 cores + Hyper Threading: >>> import psutil >>> psutil.cpu_count() @@ -200,11 +226,11 @@ CPU >>> psutil.cpu_count(logical=False) 2 - Note that this number is not equivalent to the number of CPUs the current - process can actually use. + Note that ``psutil.cpu_count()`` may not necessarily be equivalent to the + actual number of CPUs the current process can use. That can vary in case process CPU affinity has been changed, Linux cgroups - are being used or on Windows systems using processor groups or having more - than 64 CPUs. + are being used or (in case of Windows) on systems using processor groups or + having more than 64 CPUs. The number of usable CPUs can be obtained with: >>> len(psutil.Process().cpu_affinity()) @@ -237,7 +263,7 @@ CPU .. function:: cpu_freq(percpu=False) - Return CPU frequency as a nameduple including *current*, *min* and *max* + Return CPU frequency as a named tuple including *current*, *min* and *max* frequencies expressed in Mhz. On Linux *current* frequency reports the real-time value, on all other platforms it represents the nominal "fixed" value. @@ -268,16 +294,17 @@ CPU .. function:: getloadavg() Return the average system load over the last 1, 5 and 15 minutes as a tuple. - The load represents the processes which are in a runnable state, either + The "load" represents the processes which are in a runnable state, either using the CPU or waiting to use the CPU (e.g. waiting for disk I/O). On UNIX systems this relies on `os.getloadavg`_. On Windows this is emulated by using a Windows API that spawns a thread which keeps running in - background and updates the load average every 5 seconds, mimicking the UNIX - behavior. Thus, the first time this is called and for the next 5 seconds + background and updates results every 5 seconds, mimicking the UNIX behavior. + Thus, on Windows, the first time this is called and for the next 5 seconds it will return a meaningless ``(0.0, 0.0, 0.0)`` tuple. The numbers returned only make sense if related to the number of CPU cores - installed on the system. So, for instance, `3.14` on a system with 10 CPU - cores means that the system load was 31.4% percent over the last N minutes. + installed on the system. So, for instance, a value of `3.14` on a system + with 10 logical CPUs means that the system load was 31.4% percent over the + last N minutes. .. code-block:: python @@ -391,19 +418,27 @@ Disks (e.g. pseudo, memory, duplicate, inaccessible filesystems). Note that this may not be fully reliable on all systems (e.g. on BSD this parameter is ignored). - Named tuple's **fstype** field is a string which varies depending on the - platform. - On Linux it can be one of the values found in /proc/filesystems (e.g. - ``'ext3'`` for an ext3 hard drive o ``'iso9660'`` for the CD-ROM drive). - On Windows it is determined via `GetDriveType`_ and can be either - ``"removable"``, ``"fixed"``, ``"remote"``, ``"cdrom"``, ``"unmounted"`` or - ``"ramdisk"``. On macOS and BSD it is retrieved via `getfsstat`_ syscall. See `disk_usage.py`_ script providing an example usage. + Returns a list of named tuples with the following fields: + + * **device**: the device path (e.g. ``"/dev/hda1"``). On Windows this is the + drive letter (e.g. ``"C:\\"``). + * **mountpoint**: the mount point path (e.g. ``"/"``). On Windows this is the + drive letter (e.g. ``"C:\\"``). + * **fstype**: the partition filesystem (e.g. ``"ext3"`` on UNIX or ``"NTFS"`` + on Windows). + * **opts**: a comma-separated string indicating different mount options for + the drive/partition. Platform-dependent. + * **maxfile**: the maximum length a file name can have. + * **maxpath**: the maximum length a path name (directory name + base file + name) can have. - >>> import psutil - >>> psutil.disk_partitions() - [sdiskpart(device='/dev/sda3', mountpoint='/', fstype='ext4', opts='rw,errors=remount-ro'), - sdiskpart(device='/dev/sda7', mountpoint='/home', fstype='ext4', opts='rw')] + >>> import psutil + >>> psutil.disk_partitions() + [sdiskpart(device='/dev/sda3', mountpoint='/', fstype='ext4', opts='rw,errors=remount-ro', maxfile=255, maxpath=4096), + sdiskpart(device='/dev/sda7', mountpoint='/home', fstype='ext4', opts='rw', maxfile=255, maxpath=4096)] + + .. versionchanged:: 5.7.4 added *maxfile* and *maxpath* fields .. function:: disk_usage(path) @@ -538,7 +573,7 @@ Network numbers will always be increasing or remain the same, but never decrease. ``net_io_counters.cache_clear()`` can be used to invalidate the *nowrap* cache. - On machines with no network iterfaces this function will return ``None`` or + On machines with no network interfaces this function will return ``None`` or ``{}`` if *pernic* is ``True``. >>> import psutil @@ -703,7 +738,8 @@ Network system as a dictionary whose keys are the NIC names and value is a named tuple with the following fields: - - **isup**: a bool indicating whether the NIC is up and running. + - **isup**: a bool indicating whether the NIC is up and running (meaning + ethernet cable or Wi-Fi is connected). - **duplex**: the duplex communication type; it can be either :const:`NIC_DUPLEX_FULL`, :const:`NIC_DUPLEX_HALF` or :const:`NIC_DUPLEX_UNKNOWN`. @@ -722,6 +758,7 @@ Network .. versionadded:: 3.0.0 + .. versionchanged:: 5.7.3 `isup` on UNIX also checks whether the NIC is running. Sensors ------- @@ -758,7 +795,7 @@ Sensors Return hardware fans speed. Each entry is a named tuple representing a certain hardware sensor fan. - Fan speed is expressed in RPM (rounds per minute). + Fan speed is expressed in RPM (revolutions per minute). If sensors are not supported by the OS an empty dict is returned. Example:: @@ -768,7 +805,7 @@ Sensors See also `fans.py`_ and `sensors.py`_ for an example application. - Availability: Linux, macOS + Availability: Linux .. versionadded:: 5.2.0 @@ -836,7 +873,7 @@ Other system info Return users currently connected on the system as a list of named tuples including the following fields: - - **user**: the name of the user. + - **name**: the name of the user. - **terminal**: the tty or pseudo-tty associated with the user, if any, else ``None``. - **host**: the host name associated with the entry, if any. @@ -874,28 +911,28 @@ Functions .. versionchanged:: 5.6.0 PIDs are returned in sorted order -.. function:: process_iter(attrs=None, ad_value=None, new_only=False) +.. function:: process_iter(attrs=None, ad_value=None) Return an iterator yielding a :class:`Process` class instance for all running processes on the local machine. - Every instance is only created once and then cached into an internal table - which is updated every time an element is yielded. - Cached :class:`Process` instances are checked for identity so that you're - safe in case a PID has been reused by another process, in which case the - cached instance is updated. - This is preferred over :func:`psutil.pids()` for iterating over processes. - Sorting order in which processes are returned is based on their PID. + This should be preferred over :func:`psutil.pids()` to iterate over processes + as it's safe from race condition. + + Every :class:`Process` instance is only created once, and then cached for the + next time :func:`psutil.process_iter()` is called (if PID is still alive). + Also it makes sure process PIDs are not reused. + *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict()`. - If *attrs* is specified :meth:`Process.as_dict()` is called internally and - the resulting dict is stored as a ``info`` attribute which is attached to the - returned :class:`Process` instances. + If *attrs* is specified :meth:`Process.as_dict()` result will be stored as a + ``info`` attribute attached to the returned :class:`Process` instances. If *attrs* is an empty list it will retrieve all process info (slow). - If *new_only* is true this function will yield only new processes which - appeared since the last time it was called. - Example usage:: + + Sorting order in which processes are returned is based on their PID. + + Example:: >>> import psutil - >>> for proc in psutil.process_iter(attrs=['pid', 'name', 'username']): + >>> for proc in psutil.process_iter(['pid', 'name', 'username']): ... print(proc.info) ... {'name': 'systemd', 'pid': 1, 'username': 'root'} @@ -903,39 +940,19 @@ Functions {'name': 'ksoftirqd/0', 'pid': 3, 'username': 'root'} ... - Example of a dict comprehensions to create a ``{pid: info, ...}`` data - structure:: + A dict comprehensions to create a ``{pid: info, ...}`` data structure:: >>> import psutil - >>> procs = {p.pid: p.info for p in psutil.process_iter(attrs=['name', 'username'])} + >>> procs = {p.pid: p.info for p in psutil.process_iter(['name', 'username'])} >>> procs {1: {'name': 'systemd', 'username': 'root'}, 2: {'name': 'kthreadd', 'username': 'root'}, 3: {'name': 'ksoftirqd/0', 'username': 'root'}, ...} - Example showing how to filter processes by name (see also - `process filtering <#filtering-and-sorting-processes>`__ section for more - examples):: - - >>> import psutil - >>> [p.info for p in psutil.process_iter(attrs=['pid', 'name']) if 'python' in p.info['name']] - [{'name': 'python3', 'pid': 21947}, - {'name': 'python', 'pid': 23835}] - - Get new processes only (since last call):: - - >>> import psutil - >>> for proc in psutil.process_iter(attrs=['pid', 'name'], new_only=True): - ... print(proc.info) - ... - .. versionchanged:: 5.3.0 added "attrs" and "ad_value" parameters. - .. versionchanged:: - 5.7.0 added "new_only" parameter. - .. function:: pid_exists(pid) Check whether the given PID exists in the current process list. This is @@ -985,31 +1002,37 @@ Exceptions .. class:: NoSuchProcess(pid, name=None, msg=None) Raised by :class:`Process` class methods when no process with the given - *pid* is found in the current process list or when a process no longer + *pid* is found in the current process list, or when a process no longer exists. *name* is the name the process had before disappearing and gets set only if :meth:`Process.name()` was previously called. .. class:: ZombieProcess(pid, name=None, ppid=None, msg=None) This may be raised by :class:`Process` class methods when querying a zombie - process on UNIX (Windows doesn't have zombie processes). Depending on the - method called the OS may be able to succeed in retrieving the process - information or not. - Note: this is a subclass of :class:`NoSuchProcess` so if you're not - interested in retrieving zombies (e.g. when using :func:`process_iter()`) - you can ignore this exception and just catch :class:`NoSuchProcess`. + process on UNIX (Windows doesn't have zombie processes). + *name* and *ppid* attributes are available if :meth:`Process.name()` or + :meth:`Process.ppid()` methods were called before the process turned into a + zombie. + + .. note:: + + this is a subclass of :class:`NoSuchProcess` so if you're not interested + in retrieving zombies (e.g. when using :func:`process_iter()`) you can + ignore this exception and just catch :class:`NoSuchProcess`. .. versionadded:: 3.0.0 .. class:: AccessDenied(pid=None, name=None, msg=None) Raised by :class:`Process` class methods when permission to perform an - action is denied. "name" is the name of the process (may be ``None``). + action is denied due to insufficient privileges. + *name* attribute is available if :meth:`Process.name()` was previously called. .. class:: TimeoutExpired(seconds, pid=None, name=None, msg=None) - Raised by :meth:`Process.wait` if timeout expires and process is still - alive. + Raised by :meth:`Process.wait` method if timeout expires and the process is + still alive. + *name* attribute is available if :meth:`Process.name()` was previously called. Process class ------------- @@ -1180,16 +1203,15 @@ Process class >>> psutil.Process().environ() {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'GNOME_DESKTOP_SESSION_ID': 'this-is-deprecated', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} - Availability: Linux, macOS, Windows, SunOS - .. versionadded:: 4.0.0 .. versionchanged:: 5.3.0 added SunOS support .. versionchanged:: 5.6.3 added AIX suport + .. versionchanged:: 5.7.3 added BSD suport .. method:: create_time() The process creation time as a floating point number expressed in seconds - since the epoch, in UTC. The return value is cached after first call. + since the epoch. The return value is cached after first call. >>> import psutil, datetime >>> p = psutil.Process() @@ -1365,16 +1387,17 @@ Process class >>> import psutil >>> p = psutil.Process() - >>> # process may open no more than 128 file descriptors - >>> p.rlimit(psutil.RLIMIT_NOFILE, (128, 128)) - >>> # process may create files no bigger than 1024 bytes - >>> p.rlimit(psutil.RLIMIT_FSIZE, (1024, 1024)) - >>> # get - >>> p.rlimit(psutil.RLIMIT_FSIZE) + >>> p.rlimit(psutil.RLIMIT_NOFILE, (128, 128)) # process can open max 128 file descriptors + >>> p.rlimit(psutil.RLIMIT_FSIZE, (1024, 1024)) # can create files no bigger than 1024 bytes + >>> p.rlimit(psutil.RLIMIT_FSIZE) # get (1024, 1024) >>> - Availability: Linux + Also see `procinfo.py`_ script. + + Availability: Linux, FreeBSD + + .. versionchanged:: 5.7.3 added FreeBSD support .. method:: io_counters() @@ -1447,9 +1470,16 @@ Process class .. method:: threads() - Return threads opened by process as a list of named tuples including thread - id and thread CPU times (user/system). On OpenBSD this method requires - root privileges. + Return threads opened by process as a list of named tuples. On OpenBSD this + method requires root privileges. + + - **id**: the native thread ID assigned by the kernel. If :attr:`pid` refers + to the current process, this matches the + `native_id <https://docs.python.org/3/library/threading.html#threading.Thread.native_id>`__ + attribute of the `threading.Thread`_ class, and can be used to reference + individual Python threads running within your own Python app. + - **user_time**: time spent in user mode. + - **system_time**: time spent in kernel mode. .. method:: cpu_times() @@ -1461,7 +1491,7 @@ Process class - **system**: time spent in kernel mode. - **children_user**: user time of all child processes (always ``0`` on Windows and macOS). - - **system_user**: user time of all child processes (always ``0`` on + - **children_system**: system time of all child processes (always ``0`` on Windows and macOS). - **iowait**: (Linux) time spent waiting for blocking I/O to complete. This value is excluded from `user` and `system` times count (because the @@ -1663,7 +1693,7 @@ Process class (USS, PSS and swap). The additional metrics provide a better representation of "effective" process memory consumption (in case of USS) as explained in detail in this - `blog post <http://grodola.blogspot.com/2016/02/psutil-4-real-process-memory-and-environ.html>`__. + `blog post <https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python>`__. It does so by passing through the whole process address. As such it usually requires higher user privileges than :meth:`memory_info` and is considerably slower. @@ -1980,47 +2010,51 @@ Process class .. method:: wait(timeout=None) - Wait for process termination and if the process is a child of the current - one also return the exit code, else ``None``. On Windows there's - no such limitation (exit code is always returned). If the process is - already terminated immediately return ``None`` instead of raising - :class:`NoSuchProcess`. + Wait for a process PID to terminate. The details about the return value + differ on UNIX and Windows. + + *On UNIX*: if the process terminated normally, the return value is a + positive integer >= 0 indicating the exit code. + If the process was terminated by a signal return the negated value of the + signal which caused the termination (e.g. ``-SIGTERM``). + If PID is not a children of `os.getpid`_ (current process) just wait until + the process disappears and return ``None``. + If PID does not exist return ``None`` immediately. + + *On Windows*: always return the exit code, which is a positive integer as + returned by `GetExitCodeProcess`_. + *timeout* is expressed in seconds. If specified and the process is still alive raise :class:`TimeoutExpired` exception. ``timeout=0`` can be used in non-blocking apps: it will either return immediately or raise :class:`TimeoutExpired`. + + The return value is cached. To wait for multiple processes use :func:`psutil.wait_procs()`. >>> import psutil >>> p = psutil.Process(9891) >>> p.terminate() >>> p.wait() + <Negsignal.SIGTERM: -15> -Popen class ------------ + .. versionchanged:: 5.7.1 return value is cached (instead of returning + ``None``). + + .. versionchanged:: 5.7.1 on POSIX, in case of negative signal, return it + as a human readable `enum`_. .. class:: Popen(*args, **kwargs) - A more convenient interface to stdlib `subprocess.Popen`_. - It starts a sub process and you deal with it exactly as when using - `subprocess.Popen`_. - but in addition it also provides all the methods of :class:`psutil.Process` - class. - For method names common to both classes such as + Same as `subprocess.Popen`_ but in addition it provides all + :class:`psutil.Process` methods in a single class. + For the following methods which are common to both classes, psutil + implementation takes precedence: :meth:`send_signal() <psutil.Process.send_signal()>`, - :meth:`terminate() <psutil.Process.terminate()>` and - :meth:`kill() <psutil.Process.kill()>` - :class:`psutil.Process` implementation takes precedence. - For a complete documentation refer to subprocess module documentation. - - .. note:: - - Unlike `subprocess.Popen`_ this class preemptively checks whether PID has - been reused on - :meth:`send_signal() <psutil.Process.send_signal()>`, - :meth:`terminate() <psutil.Process.terminate()>` and - :meth:`kill() <psutil.Process.kill()>` - so that you can't accidentally terminate another process, fixing `BPO-6973`_. + :meth:`terminate() <psutil.Process.terminate()>`, + :meth:`kill() <psutil.Process.kill()>`. + This is done in order to avoid killing another process in case its PID has + been reused, fixing `BPO-6973`_. >>> import psutil >>> from subprocess import PIPE @@ -2036,15 +2070,6 @@ Popen class 0 >>> - :class:`psutil.Popen` objects are supported as context managers via the with - statement: on exit, standard file descriptors are closed, and the process - is waited for. This is supported on all Python versions. - - >>> import psutil, subprocess - >>> with psutil.Popen(["ifconfig"], stdout=subprocess.PIPE) as proc: - >>> log.write(proc.stdout.read()) - - .. versionchanged:: 4.4.0 added context manager support Windows services @@ -2269,29 +2294,43 @@ Process priority constants Process resources constants --------------------------- -.. data:: RLIM_INFINITY -.. data:: RLIMIT_AS -.. data:: RLIMIT_CORE -.. data:: RLIMIT_CPU -.. data:: RLIMIT_DATA -.. data:: RLIMIT_FSIZE -.. data:: RLIMIT_LOCKS -.. data:: RLIMIT_MEMLOCK -.. data:: RLIMIT_MSGQUEUE -.. data:: RLIMIT_NICE -.. data:: RLIMIT_NOFILE -.. data:: RLIMIT_NPROC -.. data:: RLIMIT_RSS -.. data:: RLIMIT_RTPRIO -.. data:: RLIMIT_RTTIME -.. data:: RLIMIT_SIGPENDING -.. data:: RLIMIT_STACK - - Constants used for getting and setting process resource limits to be used in - conjunction with :meth:`psutil.Process.rlimit()`. See `man prlimit`_ for - further information. +Linux / FreeBSD: - Availability: Linux + .. data:: RLIM_INFINITY + .. data:: RLIMIT_AS + .. data:: RLIMIT_CORE + .. data:: RLIMIT_CPU + .. data:: RLIMIT_DATA + .. data:: RLIMIT_FSIZE + .. data:: RLIMIT_MEMLOCK + .. data:: RLIMIT_NOFILE + .. data:: RLIMIT_NPROC + .. data:: RLIMIT_RSS + .. data:: RLIMIT_STACK + +Linux specific: + + .. data:: RLIMIT_LOCKS + .. data:: RLIMIT_MSGQUEUE + .. data:: RLIMIT_NICE + .. data:: RLIMIT_RTPRIO + .. data:: RLIMIT_RTTIME + .. data:: RLIMIT_SIGPENDING + +FreeBSD specific: + + .. data:: RLIMIT_SWAP + .. data:: RLIMIT_SBSIZE + .. data:: RLIMIT_NPTS + +Constants used for getting and setting process resource limits to be used in +conjunction with :meth:`psutil.Process.rlimit()`. See `resource.getrlimit`_ +for further information. + +Availability: Linux, FreeBSD + +.. versionchanged:: 5.7.3 added FreeBSD support, added ``RLIMIT_SWAP``, + ``RLIMIT_SBSIZE``, ``RLIMIT_NPTS``. Connections constants --------------------- @@ -2375,7 +2414,7 @@ Check string against :meth:`Process.name()`: def find_procs_by_name(name): "Return a list of processes matching 'name'." ls = [] - for p in psutil.process_iter(attrs=['name']): + for p in psutil.process_iter(['name']): if p.info['name'] == name: ls.append(p) return ls @@ -2391,7 +2430,7 @@ A bit more advanced, check string against :meth:`Process.name()`, def find_procs_by_name(name): "Return a list of processes matching 'name'." ls = [] - for p in psutil.process_iter(attrs=["name", "exe", "cmdline"]): + for p in psutil.process_iter(["name", "exe", "cmdline"]): if name == p.info['name'] or \ p.info['exe'] and os.path.basename(p.info['exe']) == name or \ p.info['cmdline'] and p.info['cmdline'][0] == name: @@ -2411,7 +2450,7 @@ Kill process tree timeout=None, on_terminate=None): """Kill a process tree (including grandchildren) with signal "sig" and return a (gone, still_alive) tuple. - "on_terminate", if specified, is a callabck function which is + "on_terminate", if specified, is a callback function which is called as soon as a child terminates. """ assert pid != os.getpid(), "won't kill myself" @@ -2420,7 +2459,10 @@ Kill process tree if include_parent: children.append(parent) for p in children: - p.send_signal(sig) + try: + p.send_signal(sig) + except psutil.NoSuchProcess: + pass gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) return (gone, alive) @@ -2436,21 +2478,21 @@ A collection of code samples showing how to use :func:`process_iter()` to filter Processes owned by user:: >>> import getpass - >>> pp([(p.pid, p.info['name']) for p in psutil.process_iter(attrs=['name', 'username']) if p.info['username'] == getpass.getuser()]) + >>> pp([(p.pid, p.info['name']) for p in psutil.process_iter(['name', 'username']) if p.info['username'] == getpass.getuser()]) (16832, 'bash'), (19772, 'ssh'), (20492, 'python')] Processes actively running:: - >>> pp([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'status']) if p.info['status'] == psutil.STATUS_RUNNING]) + >>> pp([(p.pid, p.info) for p in psutil.process_iter(['name', 'status']) if p.info['status'] == psutil.STATUS_RUNNING]) [(1150, {'name': 'Xorg', 'status': 'running'}), (1776, {'name': 'unity-panel-service', 'status': 'running'}), (20492, {'name': 'python', 'status': 'running'})] Processes using log files:: - >>> for p in psutil.process_iter(attrs=['name', 'open_files']): + >>> for p in psutil.process_iter(['name', 'open_files']): ... for file in p.info['open_files'] or []: ... if file.path.endswith('.log'): ... print("%-5s %-10s %s" % (p.pid, p.info['name'][:10], file.path)) @@ -2461,14 +2503,14 @@ Processes using log files:: Processes consuming more than 500M of memory:: - >>> pp([(p.pid, p.info['name'], p.info['memory_info'].rss) for p in psutil.process_iter(attrs=['name', 'memory_info']) if p.info['memory_info'].rss > 500 * 1024 * 1024]) + >>> pp([(p.pid, p.info['name'], p.info['memory_info'].rss) for p in psutil.process_iter(['name', 'memory_info']) if p.info['memory_info'].rss > 500 * 1024 * 1024]) [(2650, 'chrome', 532324352), (3038, 'chrome', 1120088064), (21915, 'sublime_text', 615407616)] Top 3 processes which consumed the most CPU time:: - >>> pp([(p.pid, p.info['name'], sum(p.info['cpu_times'])) for p in sorted(psutil.process_iter(attrs=['name', 'cpu_times']), key=lambda p: sum(p.info['cpu_times'][:2]))][-3:]) + >>> pp([(p.pid, p.info['name'], sum(p.info['cpu_times'])) for p in sorted(psutil.process_iter(['name', 'cpu_times']), key=lambda p: sum(p.info['cpu_times'][:2]))][-3:]) [(2721, 'chrome', 10219.73), (1150, 'Xorg', 11116.989999999998), (2650, 'chrome', 18451.97)] @@ -2514,10 +2556,12 @@ FAQs Unfortunately there's not much you can do about this except running the Python process with higher privileges. On Unix you may run the Python process as root or use the SUID bit - (this is the trick used by tools such as ``ps`` and ``netstat``). + (``ps`` and ``netstat`` does this). On Windows you may run the Python process as NT AUTHORITY\\SYSTEM or install - the Python script as a Windows service (this is the trick used by tools - such as ProcessHacker). + the Python script as a Windows service (ProcessHacker does this). + +* Q: is MinGW supported on Windows? +* A: no, you should Visual Studio (see `development guide`_). Running tests ============= @@ -2526,17 +2570,49 @@ Running tests $ python3 -m psutil.tests +Debug mode +========== + +If you want to debug unusual situations or want to report a bug, it may be +useful to enable debug mode via ``PSUTIL_DEBUG`` environment variable. +In this mode, psutil may (or may not) print additional information to stderr. +Usually these are error conditions which are not severe, and hence are ignored +(instead of crashing). +Unit tests automatically run with debug mode enabled. +On UNIX: + +:: + + $ PSUTIL_DEBUG=1 python3 script.py + psutil-debug [psutil/_psutil_linux.c:150]> setmntent() failed (ignored) + +On Windows: + +:: + + set PSUTIL_DEBUG=1 python.exe script.py + psutil-debug [psutil/arch/windows/process_info.c:90]> NtWow64ReadVirtualMemory64(pbi64.PebBaseAddress) -> 998 (Unknown error) (ignored) + + +Security +======== + +To report a security vulnerability, please use the `Tidelift security +contact`_. Tidelift will coordinate the fix and disclosure. + Development guide ================= -If you want to hacking on psutil (e.g. want to add a new feature or fix a bug) -take a look at the `development guide`_. +If you want to develop psutil take a look at the `development guide`_. Platforms support history ========================= +* psutil 5.8.1 (2021-10): **MidnightBSD** +* psutil 5.8.0 (2020-12): **PyPy 2** on Windows +* psutil 5.7.1 (2020-07): **Windows Nano** * psutil 5.7.0 (2020-02): drop Windows XP & Server 2003 support -* psutil 5.7.0 (2020-02): **PyPy** on Windows +* psutil 5.7.0 (2020-02): **PyPy 3** on Windows * psutil 5.4.0 (2017-11): **AIX** * psutil 3.4.1 (2016-01): **NetBSD** * psutil 3.3.0 (2015-11): **OpenBSD** @@ -2549,6 +2625,26 @@ Supported Python versions are 2.6, 2.7, 3.4+ and PyPy3. Timeline ======== +- 2020-12-19: + `5.8.0 <https://pypi.org/project/psutil/5.8.0/#files>`__ - + `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#580>`__ - + `diff <https://github.com/giampaolo/psutil/compare/release-5.7.3...release-5.8.0#files_bucket>`__ +- 2020-10-23: + `5.7.3 <https://pypi.org/project/psutil/5.7.3/#files>`__ - + `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#573>`__ - + `diff <https://github.com/giampaolo/psutil/compare/release-5.7.2...release-5.7.3#files_bucket>`__ +- 2020-07-15: + `5.7.2 <https://pypi.org/project/psutil/5.7.2/#files>`__ - + `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#572>`__ - + `diff <https://github.com/giampaolo/psutil/compare/release-5.7.1...release-5.7.2#files_bucket>`__ +- 2020-07-15: + `5.7.1 <https://pypi.org/project/psutil/5.7.1/#files>`__ - + `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#571>`__ - + `diff <https://github.com/giampaolo/psutil/compare/release-5.7.0...release-5.7.1#files_bucket>`__ +- 2020-02-18: + `5.7.0 <https://pypi.org/project/psutil/5.7.0/#files>`__ - + `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#570>`__ - + `diff <https://github.com/giampaolo/psutil/compare/release-5.6.7...release-5.7.0#files_bucket>`__ - 2019-11-26: `5.6.7 <https://pypi.org/project/psutil/5.6.7/#files>`__ - `what's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst#567>`__ - @@ -2870,13 +2966,13 @@ Timeline .. _`cpu_distribution.py`: https://github.com/giampaolo/psutil/blob/master/scripts/cpu_distribution.py .. _`development guide`: https://github.com/giampaolo/psutil/blob/master/docs/DEVGUIDE.rst .. _`disk_usage.py`: https://github.com/giampaolo/psutil/blob/master/scripts/disk_usage.py -.. _`donation`: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 -.. _`enums`: https://docs.python.org/3/library/enum.html#module-enum +.. _`enum`: https://docs.python.org/3/library/enum.html#module-enum .. _`fans.py`: https://github.com/giampaolo/psutil/blob/master/scripts/fans.py .. _`GetDriveType`: https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-getdrivetypea +.. _`GetExitCodeProcess`: https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess .. _`getfsstat`: http://www.manpagez.com/man/2/getfsstat/ .. _`GetPriorityClass`: https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-getpriorityclass -.. _`Giampaolo Rodola`: http://grodola.blogspot.com/p/about.html +.. _`Giampaolo Rodola`: https://gmpy.dev/about .. _`hash`: https://docs.python.org/3/library/functions.html#hash .. _`ifconfig.py`: https://github.com/giampaolo/psutil/blob/master/scripts/ifconfig.py .. _`ioprio_get`: https://linux.die.net/man/2/ioprio_get @@ -2903,6 +2999,7 @@ Timeline .. _`os.times`: https://docs.python.org//library/os.html#os.times .. _`pmap.py`: https://github.com/giampaolo/psutil/blob/master/scripts/pmap.py .. _`PROCESS_MEMORY_COUNTERS_EX`: https://docs.microsoft.com/en-us/windows/desktop/api/psapi/ns-psapi-_process_memory_counters_ex +.. _`procinfo.py`: https://github.com/giampaolo/psutil/blob/master/scripts/procinfo.py .. _`procsmem.py`: https://github.com/giampaolo/psutil/blob/master/scripts/procsmem.py .. _`resource.getrlimit`: https://docs.python.org/3/library/resource.html#resource.getrlimit .. _`resource.setrlimit`: https://docs.python.org/3/library/resource.html#resource.setrlimit @@ -2915,8 +3012,11 @@ Timeline .. _`SOCK_SEQPACKET`: https://docs.python.org/3/library/socket.html#socket.SOCK_SEQPACKET .. _`SOCK_STREAM`: https://docs.python.org/3/library/socket.html#socket.SOCK_STREAM .. _`socket.fromfd`: https://docs.python.org/3/library/socket.html#socket.fromfd +.. _`subprocess.Popen.wait`: https://docs.python.org/3/library/subprocess.html#subprocess.Popen.wait .. _`subprocess.Popen`: https://docs.python.org/3/library/subprocess.html#subprocess.Popen .. _`temperatures.py`: https://github.com/giampaolo/psutil/blob/master/scripts/temperatures.py .. _`TerminateProcess`: https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-terminateprocess +.. _`threading.Thread`: https://docs.python.org/3/library/threading.html#threading.Thread .. _Tidelift security contact: https://tidelift.com/security .. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme +.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme @@ -28,7 +28,7 @@ if "%PYTHON%" == "" ( ) if "%TSCRIPT%" == "" ( - set TSCRIPT=psutil\tests\__main__.py + set TSCRIPT=psutil\tests\runner.py ) rem Needed to locate the .pypirc file and upload exes on PyPI. diff --git a/psutil/__init__.py b/psutil/__init__.py index 81f1cee9..f186d90c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -21,7 +21,6 @@ Works with Python versions from 2.6 to 3.4+. """ from __future__ import division - import collections import contextlib import datetime @@ -39,9 +38,7 @@ except ImportError: from . import _common from ._common import AccessDenied -from ._common import deprecated_method from ._common import Error -from ._common import memoize from ._common import memoize_when_activated from ._common import NoSuchProcess from ._common import TimeoutExpired @@ -50,21 +47,9 @@ from ._common import ZombieProcess from ._compat import long from ._compat import PermissionError from ._compat import ProcessLookupError +from ._compat import SubprocessTimeoutExpired as _SubprocessTimeoutExpired from ._compat import PY3 as _PY3 -from ._common import STATUS_DEAD -from ._common import STATUS_DISK_SLEEP -from ._common import STATUS_IDLE -from ._common import STATUS_LOCKED -from ._common import STATUS_PARKED -from ._common import STATUS_RUNNING -from ._common import STATUS_SLEEPING -from ._common import STATUS_STOPPED -from ._common import STATUS_TRACING_STOP -from ._common import STATUS_WAITING -from ._common import STATUS_WAKING -from ._common import STATUS_ZOMBIE - from ._common import CONN_CLOSE from ._common import CONN_CLOSE_WAIT from ._common import CONN_CLOSING @@ -80,6 +65,20 @@ from ._common import CONN_TIME_WAIT from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN +from ._common import POWER_TIME_UNKNOWN +from ._common import POWER_TIME_UNLIMITED +from ._common import STATUS_DEAD +from ._common import STATUS_DISK_SLEEP +from ._common import STATUS_IDLE +from ._common import STATUS_LOCKED +from ._common import STATUS_PARKED +from ._common import STATUS_RUNNING +from ._common import STATUS_SLEEPING +from ._common import STATUS_STOPPED +from ._common import STATUS_TRACING_STOP +from ._common import STATUS_WAITING +from ._common import STATUS_WAKING +from ._common import STATUS_ZOMBIE from ._common import AIX from ._common import BSD @@ -104,44 +103,6 @@ if LINUX: from ._pslinux import IOPRIO_CLASS_IDLE # NOQA from ._pslinux import IOPRIO_CLASS_NONE # NOQA from ._pslinux import IOPRIO_CLASS_RT # NOQA - # Linux >= 2.6.36 - if _psplatform.HAS_PRLIMIT: - from ._psutil_linux import RLIM_INFINITY # NOQA - from ._psutil_linux import RLIMIT_AS # NOQA - from ._psutil_linux import RLIMIT_CORE # NOQA - from ._psutil_linux import RLIMIT_CPU # NOQA - from ._psutil_linux import RLIMIT_DATA # NOQA - from ._psutil_linux import RLIMIT_FSIZE # NOQA - from ._psutil_linux import RLIMIT_LOCKS # NOQA - from ._psutil_linux import RLIMIT_MEMLOCK # NOQA - from ._psutil_linux import RLIMIT_NOFILE # NOQA - from ._psutil_linux import RLIMIT_NPROC # NOQA - from ._psutil_linux import RLIMIT_RSS # NOQA - from ._psutil_linux import RLIMIT_STACK # NOQA - # Kinda ugly but considerably faster than using hasattr() and - # setattr() against the module object (we are at import time: - # speed matters). - from . import _psutil_linux - try: - RLIMIT_MSGQUEUE = _psutil_linux.RLIMIT_MSGQUEUE - except AttributeError: - pass - try: - RLIMIT_NICE = _psutil_linux.RLIMIT_NICE - except AttributeError: - pass - try: - RLIMIT_RTPRIO = _psutil_linux.RLIMIT_RTPRIO - except AttributeError: - pass - try: - RLIMIT_RTTIME = _psutil_linux.RLIMIT_RTTIME - except AttributeError: - pass - try: - RLIMIT_SIGPENDING = _psutil_linux.RLIMIT_SIGPENDING - except AttributeError: - pass elif WINDOWS: from . import _pswindows as _psplatform @@ -199,6 +160,7 @@ __all__ = [ "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1", "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", "CONN_NONE", + # "CONN_IDLE", "CONN_BOUND", "AF_LINK", @@ -209,6 +171,11 @@ __all__ = [ "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "MACOS", "OSX", "POSIX", "SUNOS", "WINDOWS", "AIX", + # "RLIM_INFINITY", "RLIMIT_AS", "RLIMIT_CORE", "RLIMIT_CPU", "RLIMIT_DATA", + # "RLIMIT_FSIZE", "RLIMIT_LOCKS", "RLIMIT_MEMLOCK", "RLIMIT_NOFILE", + # "RLIMIT_NPROC", "RLIMIT_RSS", "RLIMIT_STACK", "RLIMIT_MSGQUEUE", + # "RLIMIT_NICE", "RLIMIT_RTPRIO", "RLIMIT_RTTIME", "RLIMIT_SIGPENDING", + # classes "Process", "Popen", @@ -227,16 +194,30 @@ __all__ = [ __all__.extend(_psplatform.__extra__all__) + +# Linux, FreeBSD +if hasattr(_psplatform.Process, "rlimit"): + # Populate global namespace with RLIM* constants. + from . import _psutil_posix + + _globals = globals() + _name = None + for _name in dir(_psutil_posix): + if _name.startswith('RLIM') and _name.isupper(): + _globals[_name] = getattr(_psutil_posix, _name) + __all__.append(_name) + del _globals, _name + +AF_LINK = _psplatform.AF_LINK + __author__ = "Giampaolo Rodola'" -__version__ = "5.7.0" +__version__ = "5.8.1" version_info = tuple([int(num) for num in __version__.split('.')]) _timer = getattr(time, 'monotonic', time.time) -AF_LINK = _psplatform.AF_LINK -POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED -POWER_TIME_UNKNOWN = _common.POWER_TIME_UNKNOWN _TOTAL_PHYMEM = None _LOWEST_PID = None +_SENTINEL = object() # Sanity check in case the user messed up with psutil installation # or did something weird with sys.path. In this case we might end @@ -268,7 +249,7 @@ if (int(__version__.replace('.', '')) != if hasattr(_psplatform, 'ppid_map'): # Faster version (Windows and Linux). _ppid_map = _psplatform.ppid_map -else: +else: # pragma: no cover def _ppid_map(): """Return a {pid: ppid, ...} dict for all running processes in one shot. Used to speed up Process.children(). @@ -289,7 +270,11 @@ def _assert_pid_not_reused(fun): @functools.wraps(fun) def wrapper(self, *args, **kwargs): if not self.is_running(): - raise NoSuchProcess(self.pid, self._name) + if self._pid_reused: + msg = "process no longer exists and its PID has been reused" + else: + msg = None + raise NoSuchProcess(self.pid, self._name, msg=msg) return fun(self, *args, **kwargs) return wrapper @@ -360,6 +345,7 @@ class Process(object): self._exe = None self._create_time = None self._gone = False + self._pid_reused = False self._hash = None self._lock = threading.RLock() # used for caching on Windows only (on POSIX ppid may change) @@ -369,6 +355,7 @@ class Process(object): self._proc = _psplatform.Process(pid) self._last_sys_cpu_times = None self._last_proc_cpu_times = None + self._exitcode = _SENTINEL # cache creation time for later use in is_running() method try: self.create_time() @@ -383,8 +370,7 @@ class Process(object): pass except NoSuchProcess: if not _ignore_nsp: - msg = 'no process found with pid %s' % pid - raise NoSuchProcess(pid, None, msg) + raise NoSuchProcess(pid, msg='process PID not found') else: self._gone = True # This pair is supposed to indentify a Process instance @@ -396,23 +382,29 @@ class Process(object): def __str__(self): try: info = collections.OrderedDict() - except AttributeError: + except AttributeError: # pragma: no cover info = {} # Python 2.6 info["pid"] = self.pid - try: - info["name"] = self.name() + if self._name: + info['name'] = self._name + with self.oneshot(): + try: + info["name"] = self.name() + info["status"] = self.status() + except ZombieProcess: + info["status"] = "zombie" + except NoSuchProcess: + info["status"] = "terminated" + except AccessDenied: + pass + if self._exitcode not in (_SENTINEL, None): + info["exitcode"] = self._exitcode if self._create_time: info['started'] = _pprint_secs(self._create_time) - except ZombieProcess: - info["status"] = "zombie" - except NoSuchProcess: - info["status"] = "terminated" - except AccessDenied: - pass - return "%s.%s(%s)" % ( - self.__class__.__module__, - self.__class__.__name__, - ", ".join(["%s=%r" % (k, v) for k, v in info.items()])) + return "%s.%s(%s)" % ( + self.__class__.__module__, + self.__class__.__name__, + ", ".join(["%s=%r" % (k, v) for k, v in info.items()])) __repr__ = __str__ @@ -584,7 +576,7 @@ class Process(object): It also checks if PID has been reused by another process in which case return False. """ - if self._gone: + if self._gone or self._pid_reused: return False try: # Checking if PID is alive is not enough as the PID might @@ -592,7 +584,8 @@ class Process(object): # verify process identity. # Process identity / uniqueness over time is guaranteed by # (PID + creation time) and that is verified in __eq__. - return self == Process(self.pid) + self._pid_reused = self != Process(self.pid) + return not self._pid_reused except ZombieProcess: # We should never get here as it's already handled in # Process.__init__; here just for extra safety. @@ -717,7 +710,7 @@ class Process(object): def create_time(self): """The process creation time as a floating point number - expressed in seconds since the epoch, in UTC. + expressed in seconds since the epoch. The return value is cached after first call. """ if self._create_time is None: @@ -798,7 +791,7 @@ class Process(object): else: return self._proc.ionice_set(ioclass, value) - # Linux only + # Linux / FreeBSD only if hasattr(_psplatform.Process, "rlimit"): def rlimit(self, resource, limits=None): @@ -806,15 +799,12 @@ class Process(object): tuple. *resource* is one of the RLIMIT_* constants. - *limits* is supposed to be a (soft, hard) tuple. + *limits* is supposed to be a (soft, hard) tuple. See "man prlimit" for further info. - Available on Linux only. + Available on Linux and FreeBSD only. """ - if limits is None: - return self._proc.rlimit(resource) - else: - return self._proc.rlimit(resource, limits) + return self._proc.rlimit(resource, limits) # Windows, Linux and FreeBSD only if hasattr(_psplatform.Process, "cpu_affinity_get"): @@ -828,7 +818,7 @@ class Process(object): (Windows, Linux and BSD only). """ if cpus is None: - return list(set(self._proc.cpu_affinity_get())) + return sorted(set(self._proc.cpu_affinity_get())) else: if not cpus: if hasattr(self._proc, "_get_eligible_cpus"): @@ -850,7 +840,7 @@ class Process(object): """ return self._proc.cpu_num() - # Linux, macOS, Windows, Solaris, AIX + # All platforms has it, but maybe not in the future. if hasattr(_psplatform.Process, "environ"): def environ(self): @@ -1070,7 +1060,7 @@ class Process(object): """ return self._proc.memory_info() - @deprecated_method(replacement="memory_info") + @_common.deprecated_method(replacement="memory_info") def memory_info_ex(self): return self.memory_info() @@ -1273,7 +1263,18 @@ class Process(object): """ if timeout is not None and not timeout >= 0: raise ValueError("timeout must be a positive integer") - return self._proc.wait(timeout) + if self._exitcode is not _SENTINEL: + return self._exitcode + self._exitcode = self._proc.wait(timeout) + return self._exitcode + + +# The valid attr names which can be processed by Process.as_dict(). +_as_dict_attrnames = set( + [x for x in dir(Process) if not x.startswith('_') and x not in + ['send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', + 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', + 'memory_info_ex', 'oneshot']]) # ===================================================================== @@ -1282,11 +1283,17 @@ class Process(object): class Popen(Process): - """A more convenient interface to stdlib subprocess.Popen class. - It starts a sub process and deals with it exactly as when using - subprocess.Popen class but in addition also provides all the - properties and methods of psutil.Process class as a unified - interface: + """Same as subprocess.Popen, but in addition it provides all + psutil.Process methods in a single class. + For the following methods which are common to both classes, psutil + implementation takes precedence: + + * send_signal() + * terminate() + * kill() + + This is done in order to avoid killing another process in case its + PID has been reused, fixing BPO-6973. >>> import psutil >>> from subprocess import PIPE @@ -1303,17 +1310,6 @@ class Popen(Process): >>> p.wait(timeout=2) 0 >>> - - For method names common to both classes such as kill(), terminate() - and wait(), psutil.Process implementation takes precedence. - - Unlike subprocess.Popen this class pre-emptively checks whether PID - has been reused on send_signal(), terminate() and kill() so that - you don't accidentally terminate another process, fixing - http://bugs.python.org/issue6973. - - For a complete documentation refer to: - http://docs.python.org/3/library/subprocess.html """ def __init__(self, *args, **kwargs): @@ -1365,14 +1361,6 @@ class Popen(Process): return ret -# The valid attr names which can be processed by Process.as_dict(). -_as_dict_attrnames = set( - [x for x in dir(Process) if not x.startswith('_') and x not in - ['send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', - 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', - 'memory_info_ex', 'oneshot']]) - - # ===================================================================== # --- system processes related functions # ===================================================================== @@ -1405,10 +1393,9 @@ def pid_exists(pid): _pmap = {} -_lock = threading.Lock() -def process_iter(attrs=None, ad_value=None, new_only=False): +def process_iter(attrs=None, ad_value=None): """Return a generator yielding a Process instance for all running processes. @@ -1428,63 +1415,60 @@ def process_iter(attrs=None, ad_value=None, new_only=False): to returned Process instance. If *attrs* is an empty list it will retrieve all process info (slow). - If *new_only* is true this function will yield only new processes - which appeared since the last time it was called. """ + global _pmap + def add(pid): proc = Process(pid) if attrs is not None: proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) - with _lock: - _pmap[proc.pid] = proc + pmap[proc.pid] = proc return proc def remove(pid): - with _lock: - _pmap.pop(pid, None) + pmap.pop(pid, None) + pmap = _pmap.copy() a = set(pids()) - b = set(_pmap.keys()) + b = set(pmap.keys()) new_pids = a - b gone_pids = b - a for pid in gone_pids: remove(pid) - - with _lock: - ls = list(dict.fromkeys(new_pids).items()) - if not new_only: - ls += list(_pmap.items()) - ls.sort() - - for pid, proc in ls: - try: - if proc is None: # new process - yield add(pid) - else: - # use is_running() to check whether PID has been reused by - # another process in which case yield a new Process instance - if proc.is_running(): - if attrs is not None: - proc.info = proc.as_dict( - attrs=attrs, ad_value=ad_value) - yield proc - else: + try: + ls = sorted(list(pmap.items()) + list(dict.fromkeys(new_pids).items())) + for pid, proc in ls: + try: + if proc is None: # new process yield add(pid) - except NoSuchProcess: - remove(pid) - except AccessDenied: - # Process creation time can't be determined hence there's - # no way to tell whether the pid of the cached process - # has been reused. Just return the cached version. - if proc is None and pid in _pmap: - try: - yield _pmap[pid] - except KeyError: - # If we get here it is likely that 2 threads were - # using process_iter(). - pass - else: - raise + else: + # use is_running() to check whether PID has been + # reused by another process in which case yield a + # new Process instance + if proc.is_running(): + if attrs is not None: + proc.info = proc.as_dict( + attrs=attrs, ad_value=ad_value) + yield proc + else: + yield add(pid) + except NoSuchProcess: + remove(pid) + except AccessDenied: + # Process creation time can't be determined hence there's + # no way to tell whether the pid of the cached process + # has been reused. Just return the cached version. + if proc is None and pid in pmap: + try: + yield pmap[pid] + except KeyError: + # If we get here it is likely that 2 threads were + # using process_iter(). + pass + else: + raise + finally: + _pmap = pmap def wait_procs(procs, timeout=None, callback=None): @@ -1528,6 +1512,8 @@ def wait_procs(procs, timeout=None, callback=None): returncode = proc.wait(timeout=timeout) except TimeoutExpired: pass + except _SubprocessTimeoutExpired: + pass else: if returncode is not None or not proc.is_running(): # Set new Process instance attribute. @@ -1598,7 +1584,7 @@ def cpu_count(logical=True): if logical: ret = _psplatform.cpu_count_logical() else: - ret = _psplatform.cpu_count_physical() + ret = _psplatform.cpu_count_cores() if ret is not None and ret < 1: ret = None return ret @@ -1744,7 +1730,6 @@ def cpu_percent(interval=None, percpu=False): def calculate(t1, t2): times_delta = _cpu_times_deltas(t1, t2) - all_delta = _cpu_tot_time(times_delta) busy_delta = _cpu_busy_time(times_delta) @@ -1872,7 +1857,7 @@ def cpu_stats(): if hasattr(_psplatform, "cpu_freq"): def cpu_freq(percpu=False): - """Return CPU frequency as a nameduple including current, + """Return CPU frequency as a namedtuple including current, min and max frequency expressed in Mhz. If *percpu* is True and the system supports per-cpu frequency @@ -2027,7 +2012,23 @@ def disk_partitions(all=False): If *all* parameter is False return physical devices only and ignore all others. """ - return _psplatform.disk_partitions(all) + def pathconf(path, name): + try: + return os.pathconf(path, name) + except (OSError, AttributeError): + pass + + ret = _psplatform.disk_partitions(all) + if POSIX: + new = [] + for item in ret: + nt = item._replace( + maxfile=pathconf(item.mountpoint, 'PC_NAME_MAX'), + maxpath=pathconf(item.mountpoint, 'PC_PATH_MAX')) + new.append(nt) + return new + else: + return ret def disk_io_counters(perdisk=False, nowrap=True): @@ -2285,7 +2286,7 @@ if hasattr(_psplatform, "sensors_temperatures"): __all__.append("sensors_temperatures") -# Linux, macOS +# Linux if hasattr(_psplatform, "sensors_fans"): def sensors_fans(): @@ -2364,6 +2365,15 @@ if WINDOWS: # ===================================================================== +def _set_debug(value): + """Enable or disable PSUTIL_DEBUG option, which prints debugging + messages to stderr. + """ + import psutil._common + psutil._common.PSUTIL_DEBUG = bool(value) + _psplatform.cext.set_debug(bool(value)) + + def test(): # pragma: no cover from ._common import bytes2human from ._compat import get_terminal_size @@ -2372,7 +2382,7 @@ def test(): # pragma: no cover templ = "%-10s %5s %5s %7s %7s %5s %6s %6s %6s %s" attrs = ['pid', 'memory_percent', 'name', 'cmdline', 'cpu_times', 'create_time', 'memory_info', 'status', 'nice', 'username'] - print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE", + print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE", # NOQA "STATUS", "START", "TIME", "CMDLINE")) for p in process_iter(attrs, ad_value=None): if p.info['create_time']: @@ -2422,10 +2432,10 @@ def test(): # pragma: no cover ctime, cputime, cmdline) - print(line[:get_terminal_size()[0]]) + print(line[:get_terminal_size()[0]]) # NOQA -del memoize, memoize_when_activated, division, deprecated_method +del memoize_when_activated, division if sys.version_info[0] < 3: del num, x diff --git a/psutil/_common.py b/psutil/_common.py index efa8e36c..7df7c65d 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -9,6 +9,7 @@ from __future__ import division, print_function +import collections import contextlib import errno import functools @@ -18,7 +19,6 @@ import stat import sys import threading import warnings -from collections import defaultdict from collections import namedtuple from socket import AF_INET from socket import SOCK_DGRAM @@ -41,12 +41,12 @@ else: # can't take it from _common.py as this script is imported by setup.py PY3 = sys.version_info[0] == 3 +PSUTIL_DEBUG = bool(os.getenv('PSUTIL_DEBUG', 0)) __all__ = [ - # constants + # OS constants 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'MACOS', 'OSX', 'POSIX', 'SUNOS', 'WINDOWS', - 'ENCODING', 'ENCODING_ERRS', 'AF_INET6', # connection constants 'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED', 'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN', @@ -58,6 +58,8 @@ __all__ = [ 'STATUS_RUNNING', 'STATUS_SLEEPING', 'STATUS_STOPPED', 'STATUS_SUSPENDED', 'STATUS_TRACING_STOP', 'STATUS_WAITING', 'STATUS_WAKE_KILL', 'STATUS_WAKING', 'STATUS_ZOMBIE', 'STATUS_PARKED', + # other constants + 'ENCODING', 'ENCODING_ERRS', 'AF_INET6', # named tuples 'pconn', 'pcputimes', 'pctxsw', 'pgids', 'pio', 'pionice', 'popenfile', 'pthread', 'puids', 'sconn', 'scpustats', 'sdiskio', 'sdiskpart', @@ -66,7 +68,9 @@ __all__ = [ 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', 'parse_environ_block', 'path_exists_strict', 'usage_percent', 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers", - 'bytes2human', 'conn_to_ntuple', 'hilite', 'debug', + 'bytes2human', 'conn_to_ntuple', 'debug', + # shell utils + 'hilite', 'term_supports_colors', 'print_color', ] @@ -80,7 +84,7 @@ WINDOWS = os.name == "nt" LINUX = sys.platform.startswith("linux") MACOS = sys.platform.startswith("darwin") OSX = MACOS # deprecated alias -FREEBSD = sys.platform.startswith("freebsd") +FREEBSD = sys.platform.startswith(("freebsd", "midnightbsd")) OPENBSD = sys.platform.startswith("openbsd") NETBSD = sys.platform.startswith("netbsd") BSD = FREEBSD or OPENBSD or NETBSD @@ -175,7 +179,8 @@ sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', 'read_bytes', 'write_bytes', 'read_time', 'write_time']) # psutil.disk_partitions() -sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint', 'fstype', 'opts']) +sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint', 'fstype', 'opts', + 'maxfile', 'maxpath']) # psutil.disk_swaps() sdiskswap = namedtuple('sdiskswap', ['path', 'total', 'used']) # psutil.net_io_counters() @@ -273,15 +278,32 @@ class Error(Exception): """ __module__ = 'psutil' - def __init__(self, msg=""): - Exception.__init__(self, msg) - self.msg = msg + def _infodict(self, attrs): + try: + info = collections.OrderedDict() + except AttributeError: # pragma: no cover + info = {} # Python 2.6 + for name in attrs: + value = getattr(self, name, None) + if value: + info[name] = value + return info + + def __str__(self): + # invoked on `raise Error` + info = self._infodict(("pid", "ppid", "name")) + if info: + details = "(%s)" % ", ".join( + ["%s=%r" % (k, v) for k, v in info.items()]) + else: + details = None + return " ".join([x for x in (self.msg, details) if x]) def __repr__(self): - ret = "psutil.%s %s" % (self.__class__.__name__, self.msg) - return ret.strip() - - __str__ = __repr__ + # invoked on `repr(Error)` + info = self._infodict(("pid", "ppid", "name", "seconds", "msg")) + details = ", ".join(["%s=%r" % (k, v) for k, v in info.items()]) + return "psutil.%s(%s)" % (self.__class__.__name__, details) class NoSuchProcess(Error): @@ -291,19 +313,10 @@ class NoSuchProcess(Error): __module__ = 'psutil' def __init__(self, pid, name=None, msg=None): - Error.__init__(self, msg) + Error.__init__(self) self.pid = pid self.name = name - self.msg = msg - if msg is None: - if name: - details = "(pid=%s, name=%s)" % (self.pid, repr(self.name)) - else: - details = "(pid=%s)" % self.pid - self.msg = "process no longer exists " + details - - def __path__(self): - return 'xxx' + self.msg = msg or "process no longer exists" class ZombieProcess(NoSuchProcess): @@ -316,19 +329,9 @@ class ZombieProcess(NoSuchProcess): __module__ = 'psutil' def __init__(self, pid, name=None, ppid=None, msg=None): - NoSuchProcess.__init__(self, msg) - self.pid = pid + NoSuchProcess.__init__(self, pid, name, msg) self.ppid = ppid - self.name = name - self.msg = msg - if msg is None: - args = ["pid=%s" % pid] - if name: - args.append("name=%s" % repr(self.name)) - if ppid: - args.append("ppid=%s" % self.ppid) - details = "(%s)" % ", ".join(args) - self.msg = "process still exists but it's a zombie " + details + self.msg = msg or "PID still exists but it's a zombie" class AccessDenied(Error): @@ -336,17 +339,10 @@ class AccessDenied(Error): __module__ = 'psutil' def __init__(self, pid=None, name=None, msg=None): - Error.__init__(self, msg) + Error.__init__(self) self.pid = pid self.name = name - self.msg = msg - if msg is None: - if (pid is not None) and (name is not None): - self.msg = "(pid=%s, name=%s)" % (pid, repr(name)) - elif (pid is not None): - self.msg = "(pid=%s)" % self.pid - else: - self.msg = "" + self.msg = msg or "" class TimeoutExpired(Error): @@ -356,14 +352,11 @@ class TimeoutExpired(Error): __module__ = 'psutil' def __init__(self, seconds, pid=None, name=None): - Error.__init__(self, "timeout after %s seconds" % seconds) + Error.__init__(self) self.seconds = seconds self.pid = pid self.name = name - if (pid is not None) and (name is not None): - self.msg += " (pid=%s, name=%s)" % (pid, repr(name)) - elif (pid is not None): - self.msg += " (pid=%s)" % self.pid + self.msg = "timeout after %s seconds" % seconds # =================================================================== @@ -452,7 +445,13 @@ def memoize_when_activated(fun): except KeyError: # case 3: we entered oneshot() ctx but there's no cache # for this entry yet - ret = self._cache[fun] = fun(self) + ret = fun(self) + try: + self._cache[fun] = ret + except AttributeError: + # multi-threading race condition, see: + # https://github.com/giampaolo/psutil/issues/1948 + pass return ret def cache_activate(proc): @@ -623,8 +622,8 @@ class _WrapNumbers: assert name not in self.reminders assert name not in self.reminder_keys self.cache[name] = input_dict - self.reminders[name] = defaultdict(int) - self.reminder_keys[name] = defaultdict(set) + self.reminders[name] = collections.defaultdict(int) + self.reminder_keys[name] = collections.defaultdict(set) def _remove_dead_reminders(self, input_dict, name): """In case the number of keys changed between calls (e.g. a @@ -759,47 +758,91 @@ else: return s -def _term_supports_colors(file=sys.stdout): - if hasattr(_term_supports_colors, "ret"): - return _term_supports_colors.ret +# ===================================================================== +# --- shell utils +# ===================================================================== + + +@memoize +def term_supports_colors(file=sys.stdout): # pragma: no cover + if os.name == 'nt': + return True try: import curses assert file.isatty() curses.setupterm() assert curses.tigetnum("colors") > 0 except Exception: - _term_supports_colors.ret = False return False else: - _term_supports_colors.ret = True - return _term_supports_colors.ret + return True -def hilite(s, ok=True, bold=False): +def hilite(s, color=None, bold=False): # pragma: no cover """Return an highlighted version of 'string'.""" - if not _term_supports_colors(): + if not term_supports_colors(): return s attr = [] - if ok is None: # no color - pass - elif ok: # green - attr.append('32') - else: # red - attr.append('31') + colors = dict(green='32', red='91', brown='33', yellow='93', blue='34', + violet='35', lightblue='36', grey='37', darkgrey='30') + colors[None] = '29' + try: + color = colors[color] + except KeyError: + raise ValueError("invalid color %r; choose between %s" % ( + list(colors.keys()))) + attr.append(color) if bold: attr.append('1') return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) -if bool(os.getenv('PSUTIL_DEBUG', 0)): - import inspect +def print_color( + s, color=None, bold=False, file=sys.stdout): # pragma: no cover + """Print a colorized version of string.""" + if not term_supports_colors(): + print(s, file=file) # NOQA + elif POSIX: + print(hilite(s, color, bold), file=file) # NOQA + else: + import ctypes + + DEFAULT_COLOR = 7 + GetStdHandle = ctypes.windll.Kernel32.GetStdHandle + SetConsoleTextAttribute = \ + ctypes.windll.Kernel32.SetConsoleTextAttribute - def debug(msg): - """If PSUTIL_DEBUG env var is set, print a debug message to stderr.""" + colors = dict(green=2, red=4, brown=6, yellow=6) + colors[None] = DEFAULT_COLOR + try: + color = colors[color] + except KeyError: + raise ValueError("invalid color %r; choose between %r" % ( + color, list(colors.keys()))) + if bold and color <= 7: + color += 8 + + handle_id = -12 if file is sys.stderr else -11 + GetStdHandle.restype = ctypes.c_ulong + handle = GetStdHandle(handle_id) + SetConsoleTextAttribute(handle, color) + try: + print(s, file=file) # NOQA + finally: + SetConsoleTextAttribute(handle, DEFAULT_COLOR) + + +def debug(msg): + """If PSUTIL_DEBUG env var is set, print a debug message to stderr.""" + if PSUTIL_DEBUG: + import inspect fname, lineno, func_name, lines, index = inspect.getframeinfo( inspect.currentframe().f_back) - print("psutil-debug [%s:%s]> %s" % (fname, lineno, msg), + if isinstance(msg, Exception): + if isinstance(msg, (OSError, IOError, EnvironmentError)): + # ...because str(exc) may contain info about the file name + msg = "ignoring %s" % msg + else: + msg = "ignoring %r" % msg + print("psutil-debug [%s:%s]> %s" % (fname, lineno, msg), # NOQA file=sys.stderr) -else: - def debug(msg): - pass diff --git a/psutil/_compat.py b/psutil/_compat.py index a9371382..e5275f5f 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -2,26 +2,46 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Module which provides compatibility with older Python versions.""" +"""Module which provides compatibility with older Python versions. +This is more future-compatible rather than the opposite (prefer latest +Python 3 way of doing things). +""" import collections +import contextlib import errno import functools import os import sys +import types + +__all__ = [ + # constants + "PY3", + # builtins + "long", "range", "super", "unicode", "basestring", + # literals + "u", "b", + # collections module + "lru_cache", + # shutil module + "which", "get_terminal_size", + # contextlib module + "redirect_stderr", + # python 3 exceptions + "FileNotFoundError", "PermissionError", "ProcessLookupError", + "InterruptedError", "ChildProcessError", "FileExistsError"] -__all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", - "lru_cache", "which", "get_terminal_size", - "FileNotFoundError", "PermissionError", "ProcessLookupError", - "InterruptedError", "ChildProcessError", "FileExistsError"] PY3 = sys.version_info[0] == 3 +_SENTINEL = object() if PY3: long = int xrange = range unicode = str basestring = str + range = range def u(s): return s @@ -30,7 +50,7 @@ if PY3: return s.encode("latin-1") else: long = long - xrange = xrange + range = xrange unicode = unicode basestring = basestring @@ -41,6 +61,70 @@ else: return s +# --- builtins + + +# Python 3 super(). +# Taken from "future" package. +# Credit: Ryan Kelly +if PY3: + super = super +else: + _builtin_super = super + + def super(type_=_SENTINEL, type_or_obj=_SENTINEL, framedepth=1): + """Like Python 3 builtin super(). If called without any arguments + it attempts to infer them at runtime. + """ + if type_ is _SENTINEL: + f = sys._getframe(framedepth) + try: + # Get the function's first positional argument. + type_or_obj = f.f_locals[f.f_code.co_varnames[0]] + except (IndexError, KeyError): + raise RuntimeError('super() used in a function with no args') + try: + # Get the MRO so we can crawl it. + mro = type_or_obj.__mro__ + except (AttributeError, RuntimeError): + try: + mro = type_or_obj.__class__.__mro__ + except AttributeError: + raise RuntimeError('super() used in a non-newstyle class') + for type_ in mro: + # Find the class that owns the currently-executing method. + for meth in type_.__dict__.values(): + # Drill down through any wrappers to the underlying func. + # This handles e.g. classmethod() and staticmethod(). + try: + while not isinstance(meth, types.FunctionType): + if isinstance(meth, property): + # Calling __get__ on the property will invoke + # user code which might throw exceptions or + # have side effects + meth = meth.fget + else: + try: + meth = meth.__func__ + except AttributeError: + meth = meth.__get__(type_or_obj, type_) + except (AttributeError, TypeError): + continue + if meth.func_code is f.f_code: + break # found + else: + # Not found. Move onto the next class in MRO. + continue + break # found + else: + raise RuntimeError('super() called outside a method') + + # Dispatch to builtin super(). + if type_or_obj is not _SENTINEL: + return _builtin_super(type_, type_or_obj) + return _builtin_super(type_) + + # --- exceptions @@ -56,9 +140,7 @@ else: # src/future/types/exceptions/pep3151.py import platform - _singleton = object() - - def instance_checking_exception(base_exception=Exception): + def _instance_checking_exception(base_exception=Exception): def wrapped(instance_checker): class TemporaryClass(base_exception): @@ -85,30 +167,30 @@ else: return wrapped - @instance_checking_exception(EnvironmentError) + @_instance_checking_exception(EnvironmentError) def FileNotFoundError(inst): - return getattr(inst, 'errno', _singleton) == errno.ENOENT + return getattr(inst, 'errno', _SENTINEL) == errno.ENOENT - @instance_checking_exception(EnvironmentError) + @_instance_checking_exception(EnvironmentError) def ProcessLookupError(inst): - return getattr(inst, 'errno', _singleton) == errno.ESRCH + return getattr(inst, 'errno', _SENTINEL) == errno.ESRCH - @instance_checking_exception(EnvironmentError) + @_instance_checking_exception(EnvironmentError) def PermissionError(inst): - return getattr(inst, 'errno', _singleton) in ( + return getattr(inst, 'errno', _SENTINEL) in ( errno.EACCES, errno.EPERM) - @instance_checking_exception(EnvironmentError) + @_instance_checking_exception(EnvironmentError) def InterruptedError(inst): - return getattr(inst, 'errno', _singleton) == errno.EINTR + return getattr(inst, 'errno', _SENTINEL) == errno.EINTR - @instance_checking_exception(EnvironmentError) + @_instance_checking_exception(EnvironmentError) def ChildProcessError(inst): - return getattr(inst, 'errno', _singleton) == errno.ECHILD + return getattr(inst, 'errno', _SENTINEL) == errno.ECHILD - @instance_checking_exception(EnvironmentError) + @_instance_checking_exception(EnvironmentError) def FileExistsError(inst): - return getattr(inst, 'errno', _singleton) == errno.EEXIST + return getattr(inst, 'errno', _SENTINEL) == errno.EEXIST if platform.python_implementation() != "CPython": try: @@ -117,7 +199,7 @@ else: pass except OSError: raise RuntimeError( - "broken / incompatible Python implementation, see: " + "broken or incompatible Python implementation, see: " "https://github.com/giampaolo/psutil/issues/1659") @@ -343,3 +425,25 @@ except ImportError: return (res[1], res[0]) except Exception: return fallback + + +# python 3.3 +try: + from subprocess import TimeoutExpired as SubprocessTimeoutExpired +except ImportError: + class SubprocessTimeoutExpired: + pass + + +# python 3.5 +try: + from contextlib import redirect_stderr +except ImportError: + @contextlib.contextmanager + def redirect_stderr(new_target): + original = getattr(sys, "stderr") + try: + setattr(sys, "stderr", new_target) + yield new_target + finally: + setattr(sys, "stderr", original) diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 994366aa..3e3a3d14 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -46,7 +46,7 @@ HAS_THREADS = hasattr(cext, "proc_threads") HAS_NET_IO_COUNTERS = hasattr(cext, "net_io_counters") HAS_PROC_IO_COUNTERS = hasattr(cext, "proc_io_counters") -PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') +PAGE_SIZE = cext_posix.getpagesize() AF_LINK = cext_posix.AF_LINK PROC_STATUSES = { @@ -143,7 +143,7 @@ def cpu_count_logical(): return None -def cpu_count_physical(): +def cpu_count_cores(): cmd = "lsdev -Cc processor" p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -189,7 +189,9 @@ def disk_partitions(all=False): # filter by filesystem having a total size > 0. if not disk_usage(mountpoint).total: continue - ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + maxfile = maxpath = None # set later + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts, + maxfile, maxpath) retlist.append(ntuple) return retlist diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index a35e577c..fa06a772 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -51,7 +51,7 @@ if FREEBSD: cext.SWAIT: _common.STATUS_WAITING, cext.SLOCK: _common.STATUS_LOCKED, } -elif OPENBSD or NETBSD: +elif OPENBSD: PROC_STATUSES = { cext.SIDL: _common.STATUS_IDLE, cext.SSLEEP: _common.STATUS_SLEEPING, @@ -76,12 +76,11 @@ elif OPENBSD or NETBSD: elif NETBSD: PROC_STATUSES = { cext.SIDL: _common.STATUS_IDLE, - cext.SACTIVE: _common.STATUS_RUNNING, - cext.SDYING: _common.STATUS_ZOMBIE, + cext.SSLEEP: _common.STATUS_SLEEPING, cext.SSTOP: _common.STATUS_STOPPED, cext.SZOMB: _common.STATUS_ZOMBIE, - cext.SDEAD: _common.STATUS_DEAD, - cext.SSUSPENDED: _common.STATUS_SUSPENDED, # unique to NetBSD + cext.SRUN: _common.STATUS_WAKING, + cext.SONPROC: _common.STATUS_RUNNING, } TCP_STATUSES = { @@ -99,10 +98,7 @@ TCP_STATUSES = { cext.PSUTIL_CONN_NONE: _common.CONN_NONE, } -if NETBSD: - PAGESIZE = os.sysconf("SC_PAGESIZE") -else: - PAGESIZE = os.sysconf("SC_PAGE_SIZE") +PAGESIZE = cext_posix.getpagesize() AF_LINK = cext_posix.AF_LINK HAS_PER_CPU_TIMES = hasattr(cext, "per_cpu_times") @@ -260,19 +256,19 @@ def cpu_count_logical(): if OPENBSD or NETBSD: - def cpu_count_physical(): + def cpu_count_cores(): # OpenBSD and NetBSD do not implement this. return 1 if cpu_count_logical() == 1 else None else: - def cpu_count_physical(): - """Return the number of physical CPUs in the system.""" + def cpu_count_cores(): + """Return the number of CPU cores in the system.""" # From the C module we'll get an XML string similar to this: # http://manpages.ubuntu.com/manpages/precise/man4/smp.4freebsd.html # We may get None in case "sysctl kern.sched.topology_spec" # is not supported on this BSD version, in which case we'll mimic # os.cpu_count() and return None. ret = None - s = cext.cpu_count_phys() + s = cext.cpu_topology() if s is not None: # get rid of padding chars appended at the end of the string index = s.rfind("</groups>") @@ -285,8 +281,7 @@ else: # needed otherwise it will memleak root.clear() if not ret: - # If logical CPUs are 1 it's obvious we'll have only 1 - # physical CPU. + # If logical CPUs == 1 it's obvious we' have only 1 core. if cpu_count_logical() == 1: return 1 return ret @@ -336,7 +331,9 @@ def disk_partitions(all=False): partitions = cext.disk_partitions() for partition in partitions: device, mountpoint, fstype, opts = partition - ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + maxfile = maxpath = None # set later + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts, + maxfile, maxpath) retlist.append(ntuple) return retlist @@ -368,7 +365,7 @@ def net_if_stats(): for name in names: try: mtu = cext_posix.net_if_mtu(name) - isup = cext_posix.net_if_flags(name) + isup = cext_posix.net_if_is_running(name) duplex, speed = cext_posix.net_if_duplex_speed(name) except OSError as err: # https://github.com/giampaolo/psutil/issues/1279 @@ -565,10 +562,10 @@ def wrap_exceptions(fun): try: return fun(self, *args, **kwargs) except ProcessLookupError: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: + if is_zombie(self.pid): raise ZombieProcess(self.pid, self._name, self._ppid) + else: + raise NoSuchProcess(self.pid, self._name) except PermissionError: raise AccessDenied(self.pid, self._name) except OSError: @@ -590,10 +587,10 @@ def wrap_exceptions_procfs(inst): # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if not pid_exists(inst.pid): - raise NoSuchProcess(inst.pid, inst._name) - else: + if is_zombie(inst.pid): raise ZombieProcess(inst.pid, inst._name, inst._ppid) + else: + raise NoSuchProcess(inst.pid, inst._name) except PermissionError: raise AccessDenied(inst.pid, inst._name) @@ -684,6 +681,10 @@ class Process(object): return cext.proc_cmdline(self.pid) @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid) + + @wrap_exceptions def terminal(self): tty_nr = self.oneshot()[kinfo_proc_map['ttynr']] tmap = _psposix.get_terminal_map() @@ -915,3 +916,15 @@ class Process(object): @wrap_exceptions def memory_maps(self): return cext.proc_memory_maps(self.pid) + + @wrap_exceptions + def rlimit(self, resource, limits=None): + if limits is None: + return cext.proc_getrlimit(self.pid, resource) + else: + if len(limits) != 2: + raise ValueError( + "second argument must be a (soft, hard) tuple, " + "got %s" % repr(limits)) + soft, hard = limits + return cext.proc_setrlimit(self.pid, resource, soft, hard) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index f745d61a..a796386e 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -75,20 +75,13 @@ __extra__all__ = [ POWER_SUPPLY_PATH = "/sys/class/power_supply" HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) -HAS_PRLIMIT = hasattr(cext, "linux_prlimit") HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") _DEFAULT = object() -# RLIMIT_* constants, not guaranteed to be present on all kernels -if HAS_PRLIMIT: - for name in dir(cext): - if name.startswith('RLIM'): - __extra__all__.append(name) - # Number of clock ticks per second CLOCK_TICKS = os.sysconf("SC_CLK_TCK") -PAGESIZE = os.sysconf("SC_PAGE_SIZE") +PAGESIZE = cext_posix.getpagesize() BOOT_TIME = None # set later # Used when reading "big" files, namely /proc/{pid}/smaps and /proc/net/*. # On Python 2, using a buffer with open() for such files may result in a @@ -310,13 +303,61 @@ def cat(fname, fallback=_DEFAULT, binary=True): try: set_scputimes_ntuple("/proc") -except Exception: +except Exception: # pragma: no cover # Don't want to crash at import time. traceback.print_exc() scputimes = namedtuple('scputimes', 'user system idle')(0.0, 0.0, 0.0) # ===================================================================== +# --- prlimit +# ===================================================================== + +# Backport of resource.prlimit() for Python 2. Originally this was done +# in C, but CentOS-6 which we use to create manylinux wheels is too old +# and does not support prlimit() syscall. As such the resulting wheel +# would not include prlimit(), even when installed on newer systems. +# This is the only part of psutil using ctypes. + +prlimit = None +try: + from resource import prlimit # python >= 3.4 +except ImportError: + import ctypes + + libc = ctypes.CDLL(None, use_errno=True) + + if hasattr(libc, "prlimit"): + + def prlimit(pid, resource_, limits=None): + class StructRlimit(ctypes.Structure): + _fields_ = [('rlim_cur', ctypes.c_longlong), + ('rlim_max', ctypes.c_longlong)] + + current = StructRlimit() + if limits is None: + # get + ret = libc.prlimit(pid, resource_, None, ctypes.byref(current)) + else: + # set + new = StructRlimit() + new.rlim_cur = limits[0] + new.rlim_max = limits[1] + ret = libc.prlimit( + pid, resource_, ctypes.byref(new), ctypes.byref(current)) + + if ret != 0: + errno = ctypes.get_errno() + raise OSError(errno, os.strerror(errno)) + return (current.rlim_cur, current.rlim_max) + + +if prlimit is not None: + __extra__all__.extend( + [x for x in dir(cext) if x.startswith('RLIM') and x.isupper()]) + + +# ===================================================================== # --- system memory # ===================================================================== @@ -364,7 +405,6 @@ def calculate_avail_vmem(mems): if line.startswith(b'low'): watermark_low += int(line.split()[1]) watermark_low *= PAGESIZE - watermark_low = watermark_low avail = free - watermark_low pagecache = lru_active_file + lru_inactive_file @@ -619,15 +659,21 @@ def cpu_count_logical(): return num -def cpu_count_physical(): - """Return the number of physical cores in the system.""" +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" # Method #1 - core_ids = set() - for path in glob.glob( - "/sys/devices/system/cpu/cpu[0-9]*/topology/core_id"): + ls = set() + # These 2 files are the same but */core_cpus_list is newer while + # */thread_siblings_list is deprecated and may disappear in the future. + # https://www.kernel.org/doc/Documentation/admin-guide/cputopology.rst + # https://github.com/giampaolo/psutil/pull/1727#issuecomment-707624964 + # https://lkml.org/lkml/2019/2/26/41 + p1 = "/sys/devices/system/cpu/cpu[0-9]*/topology/core_cpus_list" + p2 = "/sys/devices/system/cpu/cpu[0-9]*/topology/thread_siblings_list" + for path in glob.glob(p1) or glob.glob(p2): with open_binary(path) as f: - core_ids.add(int(f.read())) - result = len(core_ids) + ls.add(f.read().strip()) + result = len(ls) if result != 0: return result @@ -639,15 +685,15 @@ def cpu_count_physical(): line = line.strip().lower() if not line: # new section - if (b'physical id' in current_info and - b'cpu cores' in current_info): + try: mapping[current_info[b'physical id']] = \ current_info[b'cpu cores'] + except KeyError: + pass current_info = {} else: # ongoing section - if (line.startswith(b'physical id') or - line.startswith(b'cpu cores')): + if line.startswith((b'physical id', b'cpu cores')): key, value = line.split(b'\t:', 1) current_info[key] = int(value) @@ -676,6 +722,17 @@ def cpu_stats(): ctx_switches, interrupts, soft_interrupts, syscalls) +def _cpu_get_cpuinfo_freq(): + """Return current CPU frequency from cpuinfo if available. + """ + ret = [] + with open_binary('%s/cpuinfo' % get_procfs_path()) as f: + for line in f: + if line.lower().startswith(b'cpu mhz'): + ret.append(float(line.split(b':', 1)[1])) + return ret + + if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or \ os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"): def cpu_freq(): @@ -683,20 +740,19 @@ if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or \ Contrarily to other OSes, Linux updates these values in real-time. """ - def get_path(num): - for p in ("/sys/devices/system/cpu/cpufreq/policy%s" % num, - "/sys/devices/system/cpu/cpu%s/cpufreq" % num): - if os.path.exists(p): - return p - + cpuinfo_freqs = _cpu_get_cpuinfo_freq() + paths = sorted( + glob.glob("/sys/devices/system/cpu/cpufreq/policy[0-9]*") or + glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq")) ret = [] - for n in range(cpu_count_logical()): - path = get_path(n) - if not path: - continue - - pjoin = os.path.join - curr = cat(pjoin(path, "scaling_cur_freq"), fallback=None) + pjoin = os.path.join + for i, path in enumerate(paths): + if len(paths) == len(cpuinfo_freqs): + # take cached value from cpuinfo if available, see: + # https://github.com/giampaolo/psutil/issues/1851 + curr = cpuinfo_freqs[i] + else: + curr = cat(pjoin(path, "scaling_cur_freq"), fallback=None) if curr is None: # Likely an old RedHat, see: # https://github.com/giampaolo/psutil/issues/1071 @@ -710,24 +766,12 @@ if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or \ ret.append(_common.scpufreq(curr, min_, max_)) return ret -elif os.path.exists("/proc/cpuinfo"): +else: def cpu_freq(): """Alternate implementation using /proc/cpuinfo. min and max frequencies are not available and are set to None. """ - ret = [] - with open_binary('%s/cpuinfo' % get_procfs_path()) as f: - for line in f: - if line.lower().startswith(b'cpu mhz'): - key, value = line.split(b'\t:', 1) - ret.append(_common.scpufreq(float(value), 0., 0.)) - return ret - -else: - def cpu_freq(): - """Dummy implementation when none of the above files are present. - """ - return [] + return [_common.scpufreq(x, 0., 0.) for x in _cpu_get_cpuinfo_freq()] # ===================================================================== @@ -791,6 +835,10 @@ class Connections: if err.errno == errno.EINVAL: # not a link continue + if err.errno == errno.ENAMETOOLONG: + # file name too long + debug(err) + continue raise else: if inode.startswith('socket:['): @@ -847,10 +895,6 @@ class Connections: else: ip = socket.inet_ntop(family, base64.b16decode(ip)) else: # IPv6 - # old version - let's keep it, just in case... - # ip = ip.decode('hex') - # return socket.inet_ntop(socket.AF_INET6, - # ''.join(ip[i:i+4][::-1] for i in xrange(0, 16, 4))) ip = base64.b16decode(ip) try: # see: https://github.com/giampaolo/psutil/issues/201 @@ -1035,12 +1079,14 @@ def net_if_stats(): for name in names: try: mtu = cext_posix.net_if_mtu(name) - isup = cext_posix.net_if_flags(name) + isup = cext_posix.net_if_is_running(name) duplex, speed = cext.net_if_duplex_speed(name) except OSError as err: # https://github.com/giampaolo/psutil/issues/1279 if err.errno != errno.ENODEV: raise + else: + debug(err) else: ret[name] = _common.snicstats(isup, duplex_map[duplex], speed, mtu) return ret @@ -1149,6 +1195,80 @@ def disk_io_counters(perdisk=False): return retdict +class RootFsDeviceFinder: + """disk_partitions() may return partitions with device == "/dev/root" + or "rootfs". This container class uses different strategies to try to + obtain the real device path. Resources: + https://bootlin.com/blog/find-root-device/ + https://www.systutorials.com/how-to-find-the-disk-where-root-is-on-in-bash-on-linux/ + """ + __slots__ = ['major', 'minor'] + + def __init__(self): + dev = os.stat("/").st_dev + self.major = os.major(dev) + self.minor = os.minor(dev) + + def ask_proc_partitions(self): + with open_text("%s/partitions" % get_procfs_path()) as f: + for line in f.readlines()[2:]: + fields = line.split() + if len(fields) < 4: # just for extra safety + continue + major = int(fields[0]) if fields[0].isdigit() else None + minor = int(fields[1]) if fields[1].isdigit() else None + name = fields[3] + if major == self.major and minor == self.minor: + if name: # just for extra safety + return "/dev/%s" % name + + def ask_sys_dev_block(self): + path = "/sys/dev/block/%s:%s/uevent" % (self.major, self.minor) + with open_text(path) as f: + for line in f: + if line.startswith("DEVNAME="): + name = line.strip().rpartition("DEVNAME=")[2] + if name: # just for extra safety + return "/dev/%s" % name + + def ask_sys_class_block(self): + needle = "%s:%s" % (self.major, self.minor) + files = glob.iglob("/sys/class/block/*/dev") + for file in files: + try: + f = open_text(file) + except FileNotFoundError: # race condition + continue + else: + with f: + data = f.read().strip() + if data == needle: + name = os.path.basename(os.path.dirname(file)) + return "/dev/%s" % name + + def find(self): + path = None + if path is None: + try: + path = self.ask_proc_partitions() + except (IOError, OSError) as err: + debug(err) + if path is None: + try: + path = self.ask_sys_dev_block() + except (IOError, OSError) as err: + debug(err) + if path is None: + try: + path = self.ask_sys_class_block() + except (IOError, OSError) as err: + debug(err) + # We use exists() because the "/dev/*" part of the path is hard + # coded, so we want to be sure. + if path is not None and os.path.exists(path): + return path + + def disk_partitions(all=False): """Return mounted disk partitions as a list of namedtuples.""" fstypes = set() @@ -1176,10 +1296,14 @@ def disk_partitions(all=False): device, mountpoint, fstype, opts = partition if device == 'none': device = '' + if device in ("/dev/root", "rootfs"): + device = RootFsDeviceFinder().find() or device if not all: if device == '' or fstype not in fstypes: continue - ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + maxfile = maxpath = None # set later + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts, + maxfile, maxpath) retlist.append(ntuple) return retlist @@ -1240,10 +1364,20 @@ def sensors_temperatures(): # https://github.com/giampaolo/psutil/issues/971 # https://github.com/nicolargo/glances/issues/1060 basenames.extend(glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*')) - basenames.extend(glob.glob( - '/sys/devices/platform/coretemp.*/hwmon/hwmon*/temp*_*')) basenames = sorted(set([x.split('_')[0] for x in basenames])) + # Only add the coretemp hwmon entries if they're not already in + # /sys/class/hwmon/ + # https://github.com/giampaolo/psutil/issues/1708 + # https://github.com/giampaolo/psutil/pull/1648 + basenames2 = glob.glob( + '/sys/devices/platform/coretemp.*/hwmon/hwmon*/temp*_*') + repl = re.compile('/sys/devices/platform/coretemp.*/hwmon/') + for name in basenames2: + altname = repl.sub('/sys/class/hwmon/', name) + if altname not in basenames: + basenames.append(name) + for base in basenames: try: path = base + '_input' @@ -1290,7 +1424,7 @@ def sensors_temperatures(): path = os.path.join(base, 'type') unit_name = cat(path, binary=False) except (IOError, OSError, ValueError) as err: - debug("ignoring %r for file %r" % (err, path)) + debug(err) continue trip_paths = glob.glob(base + '/trip_point*') @@ -1346,7 +1480,7 @@ def sensors_fans(): try: current = int(cat(base + '_input')) except (IOError, OSError) as err: - warnings.warn("ignoring %r" % err, RuntimeWarning) + debug(err) continue unit_name = cat(os.path.join(os.path.dirname(base), 'name'), binary=False) @@ -1372,10 +1506,14 @@ def sensors_battery(): for path in paths: ret = cat(path, fallback=null) if ret != null: - return int(ret) if ret.isdigit() else ret + try: + return int(ret) + except ValueError: + return ret return None - bats = [x for x in os.listdir(POWER_SUPPLY_PATH) if x.startswith('BAT')] + bats = [x for x in os.listdir(POWER_SUPPLY_PATH) if x.startswith('BAT') or + 'battery' in x.lower()] if not bats: return None # Get the first available battery. Usually this is "BAT0", except @@ -1393,12 +1531,11 @@ def sensors_battery(): energy_full = multi_cat( root + "/energy_full", root + "/charge_full") - if energy_now is None or power_now is None: - return None + time_to_empty = multi_cat(root + "/time_to_empty_now") # Percent. If we have energy_full the percentage will be more # accurate compared to reading /capacity file (float vs. int). - if energy_full is not None: + if energy_full is not None and energy_now is not None: try: percent = 100.0 * energy_now / energy_full except ZeroDivisionError: @@ -1430,11 +1567,17 @@ def sensors_battery(): # 013937745fd9050c30146290e8f963d65c0179e6/bin/battery.py#L55 if power_plugged: secsleft = _common.POWER_TIME_UNLIMITED - else: + elif energy_now is not None and power_now is not None: try: secsleft = int(energy_now / power_now * 3600) except ZeroDivisionError: secsleft = _common.POWER_TIME_UNKNOWN + elif time_to_empty is not None: + secsleft = int(time_to_empty * 60) + if secsleft < 0: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = _common.POWER_TIME_UNKNOWN return _common.sbattery(percent, secsleft, power_plugged) @@ -1763,7 +1906,7 @@ class Process(object): # According to documentation, starttime is in field 21 and the # unit is jiffies (clock ticks). # We first divide it for clock ticks and then add uptime returning - # seconds since the epoch, in UTC. + # seconds since the epoch. # Also use cached value if available. bt = BOOT_TIME or boot_time() return (ctime / CLOCK_TICKS) + bt @@ -2021,10 +2164,10 @@ class Process(object): raise ValueError("value not in 0-7 range") return cext.proc_ioprio_set(self.pid, ioclass, value) - if HAS_PRLIMIT: + if prlimit is not None: @wrap_exceptions - def rlimit(self, resource, limits=None): + def rlimit(self, resource_, limits=None): # If pid is 0 prlimit() applies to the calling process and # we don't want that. We should never get here though as # PID 0 is not supported on Linux. @@ -2033,15 +2176,14 @@ class Process(object): try: if limits is None: # get - return cext.linux_prlimit(self.pid, resource) + return prlimit(self.pid, resource_) else: # set if len(limits) != 2: raise ValueError( "second argument must be a (soft, hard) tuple, " "got %s" % repr(limits)) - soft, hard = limits - cext.linux_prlimit(self.pid, resource, soft, hard) + prlimit(self.pid, resource_, limits) except OSError as err: if err.errno == errno.ENOSYS and pid_exists(self.pid): # I saw this happening on Travis: @@ -2075,6 +2217,10 @@ class Process(object): if err.errno == errno.EINVAL: # not a link continue + if err.errno == errno.ENAMETOOLONG: + # file name too long + debug(err) + continue raise else: # If path is not an absolute there's no way to tell diff --git a/psutil/_psosx.py b/psutil/_psosx.py index e4296495..eda70d21 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -4,7 +4,6 @@ """macOS platform implementation.""" -import contextlib import errno import functools import os @@ -35,7 +34,7 @@ __extra__all__ = [] # ===================================================================== -PAGESIZE = os.sysconf("SC_PAGE_SIZE") +PAGESIZE = cext_posix.getpagesize() AF_LINK = cext_posix.AF_LINK TCP_STATUSES = { @@ -159,9 +158,9 @@ def cpu_count_logical(): return cext.cpu_count_logical() -def cpu_count_physical(): - """Return the number of physical CPUs in the system.""" - return cext.cpu_count_phys() +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + return cext.cpu_count_cores() def cpu_stats(): @@ -201,7 +200,9 @@ def disk_partitions(all=False): if not all: if not os.path.isabs(device) or not os.path.exists(device): continue - ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + maxfile = maxpath = None # set later + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts, + maxfile, maxpath) retlist.append(ntuple) return retlist @@ -262,7 +263,7 @@ def net_if_stats(): for name in names: try: mtu = cext_posix.net_if_mtu(name) - isup = cext_posix.net_if_flags(name) + isup = cext_posix.net_if_is_running(name) duplex, speed = cext_posix.net_if_duplex_speed(name) except OSError as err: # https://github.com/giampaolo/psutil/issues/1279 @@ -324,6 +325,14 @@ def pids(): pid_exists = _psposix.pid_exists +def is_zombie(pid): + try: + st = cext.proc_kinfo_oneshot(pid)[kinfo_proc_map['status']] + return st == cext.SZOMB + except Exception: + return False + + def wrap_exceptions(fun): """Decorator which translates bare OSError exceptions into NoSuchProcess and AccessDenied. @@ -333,7 +342,10 @@ def wrap_exceptions(fun): try: return fun(self, *args, **kwargs) except ProcessLookupError: - raise NoSuchProcess(self.pid, self._name) + if is_zombie(self.pid): + raise ZombieProcess(self.pid, self._name, self._ppid) + else: + raise NoSuchProcess(self.pid, self._name) except PermissionError: raise AccessDenied(self.pid, self._name) except cext.ZombieProcessError: @@ -341,32 +353,6 @@ def wrap_exceptions(fun): return wrapper -@contextlib.contextmanager -def catch_zombie(proc): - """There are some poor C APIs which incorrectly raise ESRCH when - the process is still alive or it's a zombie, or even RuntimeError - (those who don't set errno). This is here in order to solve: - https://github.com/giampaolo/psutil/issues/1044 - """ - try: - yield - except (OSError, RuntimeError) as err: - if isinstance(err, RuntimeError) or err.errno == errno.ESRCH: - try: - # status() is not supposed to lie and correctly detect - # zombies so if it raises ESRCH it's true. - status = proc.status() - except NoSuchProcess: - raise err - else: - if status == _common.STATUS_ZOMBIE: - raise ZombieProcess(proc.pid, proc._name, proc._ppid) - else: - raise AccessDenied(proc.pid, proc._name) - else: - raise - - class Process(object): """Wrapper class around underlying C implementation.""" @@ -389,8 +375,7 @@ class Process(object): @memoize_when_activated def _get_pidtaskinfo(self): # Note: should work for PIDs owned by user only. - with catch_zombie(self): - ret = cext.proc_pidtaskinfo_oneshot(self.pid) + ret = cext.proc_pidtaskinfo_oneshot(self.pid) assert len(ret) == len(pidtaskinfo_map) return ret @@ -409,18 +394,15 @@ class Process(object): @wrap_exceptions def exe(self): - with catch_zombie(self): - return cext.proc_exe(self.pid) + return cext.proc_exe(self.pid) @wrap_exceptions def cmdline(self): - with catch_zombie(self): - return cext.proc_cmdline(self.pid) + return cext.proc_cmdline(self.pid) @wrap_exceptions def environ(self): - with catch_zombie(self): - return parse_environ_block(cext.proc_environ(self.pid)) + return parse_environ_block(cext.proc_environ(self.pid)) @wrap_exceptions def ppid(self): @@ -429,8 +411,7 @@ class Process(object): @wrap_exceptions def cwd(self): - with catch_zombie(self): - return cext.proc_cwd(self.pid) + return cext.proc_cwd(self.pid) @wrap_exceptions def uids(self): @@ -503,8 +484,7 @@ class Process(object): if self.pid == 0: return [] files = [] - with catch_zombie(self): - rawlist = cext.proc_open_files(self.pid) + rawlist = cext.proc_open_files(self.pid) for path, fd in rawlist: if isfile_strict(path): ntuple = _common.popenfile(path, fd) @@ -517,8 +497,7 @@ class Process(object): raise ValueError("invalid %r kind argument; choose between %s" % (kind, ', '.join([repr(x) for x in conn_tmap]))) families, types = conn_tmap[kind] - with catch_zombie(self): - rawlist = cext.proc_connections(self.pid, families, types) + rawlist = cext.proc_connections(self.pid, families, types) ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item @@ -531,8 +510,7 @@ class Process(object): def num_fds(self): if self.pid == 0: return 0 - with catch_zombie(self): - return cext.proc_num_fds(self.pid) + return cext.proc_num_fds(self.pid) @wrap_exceptions def wait(self, timeout=None): @@ -540,13 +518,11 @@ class Process(object): @wrap_exceptions def nice_get(self): - with catch_zombie(self): - return cext_posix.getpriority(self.pid) + return cext_posix.getpriority(self.pid) @wrap_exceptions def nice_set(self, value): - with catch_zombie(self): - return cext_posix.setpriority(self.pid, value) + return cext_posix.setpriority(self.pid, value) @wrap_exceptions def status(self): diff --git a/psutil/_psposix.py b/psutil/_psposix.py index 88213ef8..706dab9a 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -6,6 +6,7 @@ import glob import os +import signal import sys import time @@ -21,6 +22,11 @@ from ._compat import ProcessLookupError from ._compat import PY3 from ._compat import unicode +if sys.version_info >= (3, 4): + import enum +else: + enum = None + __all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map'] @@ -47,66 +53,108 @@ def pid_exists(pid): return True -def wait_pid(pid, timeout=None, proc_name=None): - """Wait for process with pid 'pid' to terminate and return its - exit status code as an integer. +# Python 3.5 signals enum (contributed by me ^^): +# https://bugs.python.org/issue21076 +if enum is not None and hasattr(signal, "Signals"): + Negsignal = enum.IntEnum( + 'Negsignal', dict([(x.name, -x.value) for x in signal.Signals])) + + def negsig_to_enum(num): + """Convert a negative signal value to an enum.""" + try: + return Negsignal(num) + except ValueError: + return num +else: # pragma: no cover + def negsig_to_enum(num): + return num + + +def wait_pid(pid, timeout=None, proc_name=None, + _waitpid=os.waitpid, + _timer=getattr(time, 'monotonic', time.time), + _min=min, + _sleep=time.sleep, + _pid_exists=pid_exists): + """Wait for a process PID to terminate. + + If the process terminated normally by calling exit(3) or _exit(2), + or by returning from main(), the return value is the positive integer + passed to *exit(). + + If it was terminated by a signal it returns the negated value of the + signal which caused the termination (e.g. -SIGTERM). - If pid is not a children of os.getpid() (current process) just - waits until the process disappears and return None. + If PID is not a children of os.getpid() (current process) just + wait until the process disappears and return None. - If pid does not exist at all return None immediately. + If PID does not exist at all return None immediately. - Raise TimeoutExpired on timeout expired. + If *timeout* != None and process is still alive raise TimeoutExpired. + timeout=0 is also possible (either return immediately or raise). """ - def check_timeout(delay): + if pid <= 0: + raise ValueError("can't wait for PID 0") # see "man waitpid" + interval = 0.0001 + flags = 0 + if timeout is not None: + flags |= os.WNOHANG + stop_at = _timer() + timeout + + def sleep(interval): + # Sleep for some time and return a new increased interval. if timeout is not None: - if timer() >= stop_at: + if _timer() >= stop_at: raise TimeoutExpired(timeout, pid=pid, name=proc_name) - time.sleep(delay) - return min(delay * 2, 0.04) - - timer = getattr(time, 'monotonic', time.time) - if timeout is not None: - def waitcall(): - return os.waitpid(pid, os.WNOHANG) - stop_at = timer() + timeout - else: - def waitcall(): - return os.waitpid(pid, 0) + _sleep(interval) + return _min(interval * 2, 0.04) - delay = 0.0001 + # See: https://linux.die.net/man/2/waitpid while True: try: - retpid, status = waitcall() + retpid, status = os.waitpid(pid, flags) except InterruptedError: - delay = check_timeout(delay) + interval = sleep(interval) except ChildProcessError: # This has two meanings: - # - pid is not a child of os.getpid() in which case + # - PID is not a child of os.getpid() in which case # we keep polling until it's gone - # - pid never existed in the first place + # - PID never existed in the first place # In both cases we'll eventually return None as we # can't determine its exit status code. - while True: - if pid_exists(pid): - delay = check_timeout(delay) - else: - return + while _pid_exists(pid): + interval = sleep(interval) + return else: if retpid == 0: - # WNOHANG was used, pid is still running - delay = check_timeout(delay) + # WNOHANG flag was used and PID is still running. + interval = sleep(interval) continue - # process exited due to a signal; return the integer of - # that signal - if os.WIFSIGNALED(status): - return -os.WTERMSIG(status) - # process exited using exit(2) system call; return the - # integer exit(2) system call has been called with elif os.WIFEXITED(status): + # Process terminated normally by calling exit(3) or _exit(2), + # or by returning from main(). The return value is the + # positive integer passed to *exit(). return os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + # Process exited due to a signal. Return the negative value + # of that signal. + return negsig_to_enum(-os.WTERMSIG(status)) + # elif os.WIFSTOPPED(status): + # # Process was stopped via SIGSTOP or is being traced, and + # # waitpid() was called with WUNTRACED flag. PID is still + # # alive. From now on waitpid() will keep returning (0, 0) + # # until the process state doesn't change. + # # It may make sense to catch/enable this since stopped PIDs + # # ignore SIGTERM. + # interval = sleep(interval) + # continue + # elif os.WIFCONTINUED(status): + # # Process was resumed via SIGCONT and waitpid() was called + # # with WCONTINUED flag. + # interval = sleep(interval) + # continue else: - # should never happen + # Should never happen. raise ValueError("unknown process exit status %r" % status) @@ -119,7 +167,7 @@ def disk_usage(path): """ if PY3: st = os.statvfs(path) - else: + else: # pragma: no cover # os.statvfs() does not support unicode on Python 2: # - https://github.com/giampaolo/psutil/issues/416 # - http://bugs.python.org/issue18695 diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 62362b89..355a7623 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -43,7 +43,7 @@ __extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] # ===================================================================== -PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') +PAGE_SIZE = cext_posix.getpagesize() AF_LINK = cext_posix.AF_LINK IS_64_BIT = sys.maxsize > 2**32 @@ -155,7 +155,7 @@ def swap_memory(): total = free = 0 for line in lines: line = line.split() - t, f = line[-2:] + t, f = line[3:5] total += int(int(t) * 512) free += int(int(f) * 512) used = total - free @@ -190,9 +190,9 @@ def cpu_count_logical(): return None -def cpu_count_physical(): - """Return the number of physical CPUs in the system.""" - return cext.cpu_count_phys() +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + return cext.cpu_count_cores() def cpu_stats(): @@ -231,9 +231,11 @@ def disk_partitions(all=False): continue except OSError as err: # https://github.com/giampaolo/psutil/issues/1674 - debug("skipping %r: %r" % (mountpoint, err)) + debug("skipping %r: %s" % (mountpoint, err)) continue - ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + maxfile = maxpath = None # set later + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts, + maxfile, maxpath) retlist.append(ntuple) return retlist diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index b8584f26..ba638641 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -15,7 +15,7 @@ * - psutil.Process.io_counters read count is always 0 * - psutil.Process.io_counters may not be available on older AIX versions * - psutil.Process.threads may not be available on older AIX versions - # - psutil.net_io_counters may not be available on older AIX versions + * - psutil.net_io_counters may not be available on older AIX versions * - reading basic process info may fail or return incorrect values when * process is starting (see IBM APAR IV58499 - fixed in newer AIX versions) * - sockets and pipes may not be counted in num_fds (fixed in newer AIX @@ -679,7 +679,7 @@ psutil_net_if_stats(PyObject* self, PyObject* args) { if (sock == -1) goto error; - strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + PSUTIL_STRNCPY(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); // is up? ret = ioctl(sock, SIOCGIFFLAGS, &ifr); @@ -876,7 +876,7 @@ error: static PyObject * psutil_virtual_mem(PyObject *self, PyObject *args) { int rc; - int pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); perfstat_memory_total_t memory; rc = perfstat_memory_total( @@ -902,7 +902,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { static PyObject * psutil_swap_mem(PyObject *self, PyObject *args) { int rc; - int pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); perfstat_memory_total_t memory; rc = perfstat_memory_total( @@ -1039,8 +1039,8 @@ PsutilMethods[] = "Return CPU statistics"}, // --- others - {"set_testing", psutil_set_testing, METH_NOARGS, - "Set psutil in testing mode"}, + {"set_debug", psutil_set_debug, METH_VARARGS, + "Enable or disable PSUTIL_DEBUG messages"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 32e59571..db29a5a1 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -57,11 +57,13 @@ #include <net/route.h> #include <netinet/in.h> // process open files/connections #include <sys/un.h> +#include <kvm.h> #include "_psutil_common.h" #include "_psutil_posix.h" #ifdef PSUTIL_FREEBSD + #include "arch/freebsd/cpu.h" #include "arch/freebsd/specific.h" #include "arch/freebsd/sys_socks.h" #include "arch/freebsd/proc_socks.h" @@ -188,7 +190,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { long memstack; int oncpu; kinfo_proc kp; - long pagesize = sysconf(_SC_PAGESIZE); + long pagesize = psutil_getpagesize(); char str[1000]; PyObject *py_name; PyObject *py_ppid; @@ -392,6 +394,140 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { /* + * Return process environment as a Python dictionary + */ +PyObject * +psutil_proc_environ(PyObject *self, PyObject *args) { + int i, cnt = -1; + long pid; + char *s, **envs, errbuf[_POSIX2_LINE_MAX]; + PyObject *py_value=NULL, *py_retdict=NULL; + kvm_t *kd; +#ifdef PSUTIL_NETBSD + struct kinfo_proc2 *p; +#else + struct kinfo_proc *p; +#endif + + if (!PyArg_ParseTuple(args, "l", &pid)) + return NULL; + +#if defined(PSUTIL_FREEBSD) + kd = kvm_openfiles(NULL, "/dev/null", NULL, 0, errbuf); +#else + kd = kvm_openfiles(NULL, NULL, NULL, KVM_NO_FILES, errbuf); +#endif + if (!kd) { + convert_kvm_err("kvm_openfiles", errbuf); + return NULL; + } + + py_retdict = PyDict_New(); + if (!py_retdict) + goto error; + +#if defined(PSUTIL_FREEBSD) + p = kvm_getprocs(kd, KERN_PROC_PID, pid, &cnt); +#elif defined(PSUTIL_OPENBSD) + p = kvm_getprocs(kd, KERN_PROC_PID, pid, sizeof(*p), &cnt); +#elif defined(PSUTIL_NETBSD) + p = kvm_getproc2(kd, KERN_PROC_PID, pid, sizeof(*p), &cnt); +#endif + if (!p) { + NoSuchProcess("kvm_getprocs"); + goto error; + } + if (cnt <= 0) { + NoSuchProcess(cnt < 0 ? kvm_geterr(kd) : "kvm_getprocs: cnt==0"); + goto error; + } + + // On *BSD kernels there are a few kernel-only system processes without an + // environment (See e.g. "procstat -e 0 | 1 | 2 ..." on FreeBSD.) + // Some system process have no stats attached at all + // (they are marked with P_SYSTEM.) + // On FreeBSD, it's possible that the process is swapped or paged out, + // then there no access to the environ stored in the process' user area. + // On NetBSD, we cannot call kvm_getenvv2() for a zombie process. + // To make unittest suite happy, return an empty environment. +#if defined(PSUTIL_FREEBSD) +#if (defined(__FreeBSD_version) && __FreeBSD_version >= 700000) + if (!((p)->ki_flag & P_INMEM) || ((p)->ki_flag & P_SYSTEM)) { +#else + if ((p)->ki_flag & P_SYSTEM) { +#endif +#elif defined(PSUTIL_NETBSD) + if ((p)->p_stat == SZOMB) { +#elif defined(PSUTIL_OPENBSD) + if ((p)->p_flag & P_SYSTEM) { +#endif + kvm_close(kd); + return py_retdict; + } + +#if defined(PSUTIL_NETBSD) + envs = kvm_getenvv2(kd, p, 0); +#else + envs = kvm_getenvv(kd, p, 0); +#endif + if (!envs) { + // Map to "psutil" general high-level exceptions + switch (errno) { + case 0: + // Process has cleared it's environment, return empty one + kvm_close(kd); + return py_retdict; + case EPERM: + AccessDenied("kvm_getenvv -> EPERM"); + break; + case ESRCH: + NoSuchProcess("kvm_getenvv -> ESRCH"); + break; +#if defined(PSUTIL_FREEBSD) + case ENOMEM: + // Unfortunately, under FreeBSD kvm_getenvv() returns + // failure for certain processes ( e.g. try + // "sudo procstat -e <pid of your XOrg server>".) + // Map the error condition to 'AccessDenied'. + sprintf(errbuf, + "kvm_getenvv(pid=%ld, ki_uid=%d) -> ENOMEM", + pid, p->ki_uid); + AccessDenied(errbuf); + break; +#endif + default: + sprintf(errbuf, "kvm_getenvv(pid=%ld)", pid); + PyErr_SetFromOSErrnoWithSyscall(errbuf); + break; + } + goto error; + } + + for (i = 0; envs[i] != NULL; i++) { + s = strchr(envs[i], '='); + if (!s) + continue; + *s++ = 0; + py_value = PyUnicode_DecodeFSDefault(s); + if (!py_value) + goto error; + if (PyDict_SetItemString(py_retdict, envs[i], py_value)) { + goto error; + } + Py_DECREF(py_value); + } + + kvm_close(kd); + return py_retdict; + +error: + Py_XDECREF(py_value); + Py_XDECREF(py_retdict); + kvm_close(kd); + return NULL; +} + +/* * Return the number of logical CPUs in the system. * XXX this could be shared with macOS */ @@ -617,8 +753,10 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",softdep", sizeof(opts)); if (flags & MNT_NOSYMFOLLOW) strlcat(opts, ",nosymfollow", sizeof(opts)); +#ifdef MNT_GJOURNAL if (flags & MNT_GJOURNAL) strlcat(opts, ",gjournal", sizeof(opts)); +#endif if (flags & MNT_MULTILABEL) strlcat(opts, ",multilabel", sizeof(opts)); if (flags & MNT_ACLS) @@ -627,8 +765,10 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",noclusterr", sizeof(opts)); if (flags & MNT_NOCLUSTERW) strlcat(opts, ",noclusterw", sizeof(opts)); +#ifdef MNT_NFS4ACLS if (flags & MNT_NFS4ACLS) strlcat(opts, ",nfs4acls", sizeof(opts)); +#endif #elif PSUTIL_NETBSD if (flags & MNT_NODEV) strlcat(opts, ",nodev", sizeof(opts)); @@ -831,7 +971,7 @@ psutil_users(PyObject *self, PyObject *args) { py_tty, // tty py_hostname, // hostname (float)ut.ut_time, // start time -#ifdef PSUTIL_OPENBSD +#if defined(PSUTIL_OPENBSD) || (defined(__FreeBSD_version) && __FreeBSD_version < 900000) -1 // process id (set to None later) #else ut.ut_pid // TODO: use PyLong_FromPid @@ -953,9 +1093,15 @@ static PyMethodDef mod_methods[] = { "Return process CPU affinity."}, {"proc_cpu_affinity_set", psutil_proc_cpu_affinity_set, METH_VARARGS, "Set process CPU affinity."}, - {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, - "Return an XML string to determine the number physical CPUs."}, + {"proc_getrlimit", psutil_proc_getrlimit, METH_VARARGS, + "Get process resource limits."}, + {"proc_setrlimit", psutil_proc_setrlimit, METH_VARARGS, + "Set process resource limits."}, + {"cpu_topology", psutil_cpu_topology, METH_VARARGS, + "Return CPU topology as an XML string."}, #endif + {"proc_environ", psutil_proc_environ, METH_VARARGS, + "Return process environment"}, // --- system-related functions @@ -1004,8 +1150,8 @@ static PyMethodDef mod_methods[] = { #endif // --- others - {"set_testing", psutil_set_testing, METH_NOARGS, - "Set psutil in testing mode"}, + {"set_debug", psutil_set_debug, METH_VARARGS, + "Enable or disable PSUTIL_DEBUG messages"}, {NULL, NULL, 0, NULL} }; @@ -1066,7 +1212,9 @@ static PyMethodDef mod_methods[] = { if (PyModule_AddIntConstant(mod, "SSLEEP", LSSLEEP)) INITERR; if (PyModule_AddIntConstant(mod, "SSTOP", LSSTOP)) INITERR; if (PyModule_AddIntConstant(mod, "SZOMB", LSZOMB)) INITERR; +#if __NetBSD_Version__ < 500000000 if (PyModule_AddIntConstant(mod, "SDEAD", LSDEAD)) INITERR; +#endif if (PyModule_AddIntConstant(mod, "SONPROC", LSONPROC)) INITERR; // unique to NetBSD if (PyModule_AddIntConstant(mod, "SSUSPENDED", LSSUSPENDED)) INITERR; diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 07578eda..6ed14658 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -14,8 +14,6 @@ // ==================================================================== int PSUTIL_DEBUG = 0; -int PSUTIL_TESTING = 0; -// PSUTIL_CONN_NONE // ==================================================================== @@ -23,9 +21,8 @@ int PSUTIL_TESTING = 0; // ==================================================================== // PyPy on Windows -#if defined(PSUTIL_WINDOWS) && \ - defined(PYPY_VERSION) && \ - !defined(PyErr_SetFromWindowsErrWithFilename) +#if defined(PSUTIL_WINDOWS) && defined(PYPY_VERSION) +#if !defined(PyErr_SetFromWindowsErrWithFilename) PyObject * PyErr_SetFromWindowsErrWithFilename(int winerr, const char *filename) { PyObject *py_exc = NULL; @@ -58,7 +55,17 @@ error: Py_XDECREF(py_winerr); return NULL; } -#endif // PYPY on Windows +#endif // !defined(PyErr_SetFromWindowsErrWithFilename) + + +// PyPy 2.7 +#if !defined(PyErr_SetFromWindowsErr) +PyObject * +PyErr_SetFromWindowsErr(int winerr) { + return PyErr_SetFromWindowsErrWithFilename(winerr, ""); +} +#endif // !defined(PyErr_SetFromWindowsErr) +#endif // defined(PSUTIL_WINDOWS) && defined(PYPY_VERSION) // ==================================================================== @@ -74,8 +81,9 @@ PyErr_SetFromOSErrnoWithSyscall(const char *syscall) { char fullmsg[1024]; #ifdef PSUTIL_WINDOWS + DWORD dwLastError = GetLastError(); sprintf(fullmsg, "(originated from %s)", syscall); - PyErr_SetFromWindowsErrWithFilename(GetLastError(), fullmsg); + PyErr_SetFromWindowsErrWithFilename(dwLastError, fullmsg); #else PyObject *exc; sprintf(fullmsg, "%s (originated from %s)", strerror(errno), syscall); @@ -96,7 +104,7 @@ NoSuchProcess(const char *syscall) { PyObject *exc; char msg[1024]; - sprintf(msg, "No such process (originated from %s)", syscall); + sprintf(msg, "assume no such process (originated from %s)", syscall); exc = PyObject_CallFunction(PyExc_OSError, "(is)", ESRCH, msg); PyErr_SetObject(PyExc_OSError, exc); Py_XDECREF(exc); @@ -113,7 +121,7 @@ AccessDenied(const char *syscall) { PyObject *exc; char msg[1024]; - sprintf(msg, "Access denied (originated from %s)", syscall); + sprintf(msg, "assume access denied (originated from %s)", syscall); exc = PyObject_CallFunction(PyExc_OSError, "(is)", EACCES, msg); PyErr_SetObject(PyExc_OSError, exc); Py_XDECREF(exc); @@ -125,32 +133,26 @@ AccessDenied(const char *syscall) { // --- Global utils // ==================================================================== -/* - * Enable testing mode. This has the same effect as setting PSUTIL_TESTING - * env var. This dual method exists because updating os.environ on - * Windows has no effect. Called on unit tests setup. - */ -PyObject * -psutil_set_testing(PyObject *self, PyObject *args) { - PSUTIL_TESTING = 1; - Py_INCREF(Py_None); - return Py_None; -} +// Enable or disable PSUTIL_DEBUG messages. +PyObject * +psutil_set_debug(PyObject *self, PyObject *args) { + PyObject *value; + int x; -/* - * Print a debug message on stderr. No-op if PSUTIL_DEBUG env var is not set. - */ -void -psutil_debug(const char* format, ...) { - va_list argptr; - if (PSUTIL_DEBUG) { - va_start(argptr, format); - fprintf(stderr, "psutil-debug> "); - vfprintf(stderr, format, argptr); - fprintf(stderr, "\n"); - va_end(argptr); + if (!PyArg_ParseTuple(args, "O", &value)) + return NULL; + x = PyObject_IsTrue(value); + if (x < 0) { + return NULL; } + else if (x == 0) { + PSUTIL_DEBUG = 0; + } + else { + PSUTIL_DEBUG = 1; + } + Py_RETURN_NONE; } @@ -161,12 +163,30 @@ int psutil_setup(void) { if (getenv("PSUTIL_DEBUG") != NULL) PSUTIL_DEBUG = 1; - if (getenv("PSUTIL_TESTING") != NULL) - PSUTIL_TESTING = 1; return 0; } +// ============================================================================ +// Utility functions (BSD) +// ============================================================================ + +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) +void +convert_kvm_err(const char *syscall, char *errbuf) { + char fullmsg[8192]; + + sprintf(fullmsg, "(originated from %s: %s)", syscall, errbuf); + if (strstr(errbuf, "Permission denied") != NULL) + AccessDenied(fullmsg); + else if (strstr(errbuf, "Operation not permitted") != NULL) + AccessDenied(fullmsg); + else + PyErr_Format(PyExc_RuntimeError, fullmsg); +} +#endif + + // ==================================================================== // --- Windows // ==================================================================== @@ -179,13 +199,6 @@ int PSUTIL_WINVER; SYSTEM_INFO PSUTIL_SYSTEM_INFO; CRITICAL_SECTION PSUTIL_CRITICAL_SECTION; -#define NT_FACILITY_MASK 0xfff -#define NT_FACILITY_SHIFT 16 -#define NT_FACILITY(Status) \ - ((((ULONG)(Status)) >> NT_FACILITY_SHIFT) & NT_FACILITY_MASK) -#define NT_NTWIN32(status) (NT_FACILITY(Status) == FACILITY_WIN32) -#define WIN32_FROM_NTSTATUS(Status) (((ULONG)(Status)) & 0xffff) - // A wrapper around GetModuleHandle and GetProcAddress. PVOID @@ -264,10 +277,6 @@ psutil_loadlibs() { "ntdll.dll", "NtSetInformationProcess"); if (! NtSetInformationProcess) return 1; - WinStationQueryInformationW = psutil_GetProcAddressFromLib( - "winsta.dll", "WinStationQueryInformationW"); - if (! WinStationQueryInformationW) - return 1; NtQueryObject = psutil_GetProcAddressFromLib( "ntdll.dll", "NtQueryObject"); if (! NtQueryObject) @@ -320,6 +329,13 @@ psutil_loadlibs() { // minumum requirement: Win 7 GetLogicalProcessorInformationEx = psutil_GetProcAddressFromLib( "kernel32", "GetLogicalProcessorInformationEx"); + // minimum requirements: Windows Server Core + WTSEnumerateSessionsW = psutil_GetProcAddressFromLib( + "wtsapi32.dll", "WTSEnumerateSessionsW"); + WTSQuerySessionInformationW = psutil_GetProcAddressFromLib( + "wtsapi32.dll", "WTSQuerySessionInformationW"); + WTSFreeMemory = psutil_GetProcAddressFromLib( + "wtsapi32.dll", "WTSFreeMemory"); PyErr_Clear(); return 0; @@ -391,7 +407,6 @@ double psutil_FiletimeToUnixTime(FILETIME ft) { return _to_unix_time((ULONGLONG)ft.dwHighDateTime, (ULONGLONG)ft.dwLowDateTime); - } diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 34c428c0..6cf19d65 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -10,11 +10,15 @@ // --- Global vars / constants // ==================================================================== -extern int PSUTIL_TESTING; extern int PSUTIL_DEBUG; // a signaler for connections without an actual status static const int PSUTIL_CONN_NONE = 128; +// strncpy() variant which appends a null terminator. +#define PSUTIL_STRNCPY(dst, src, n) \ + strncpy(dst, src, n - 1); \ + dst[n - 1] = '\0' + // ==================================================================== // --- Backward compatibility with missing Python.h APIs // ==================================================================== @@ -95,10 +99,25 @@ PyObject* PyErr_SetFromOSErrnoWithSyscall(const char *syscall); // --- Global utils // ==================================================================== -PyObject* psutil_set_testing(PyObject *self, PyObject *args); -void psutil_debug(const char* format, ...); +PyObject* psutil_set_debug(PyObject *self, PyObject *args); int psutil_setup(void); + +// Print a debug message on stderr. +#define psutil_debug(...) do { \ + if (! PSUTIL_DEBUG) \ + break; \ + fprintf(stderr, "psutil-debug [%s:%d]> ", __FILE__, __LINE__); \ + fprintf(stderr, __VA_ARGS__); \ + fprintf(stderr, "\n");} while(0) + + +// ==================================================================== +// --- BSD +// ==================================================================== + +void convert_kvm_err(const char *syscall, char *errbuf); + // ==================================================================== // --- Windows // ==================================================================== @@ -123,6 +142,14 @@ int psutil_setup(void); #define MALLOC_ZERO(x) HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, (x)) #define FREE(x) HeapFree(GetProcessHeap(), 0, (x)) + #define _NT_FACILITY_MASK 0xfff + #define _NT_FACILITY_SHIFT 16 + #define _NT_FACILITY(status) \ + ((((ULONG)(status)) >> _NT_FACILITY_SHIFT) & _NT_FACILITY_MASK) + + #define NT_NTWIN32(status) (_NT_FACILITY(status) == FACILITY_WIN32) + #define WIN32_FROM_NTSTATUS(status) (((ULONG)(status)) & 0xffff) + #define LO_T 1e-7 #define HI_T 429.4967296 diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index ec8b433a..bb213610 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -23,6 +23,7 @@ #include <sys/socket.h> #include <linux/sockios.h> #include <linux/if.h> +#include <sys/resource.h> // see: https://github.com/giampaolo/psutil/issues/659 #ifdef PSUTIL_ETHTOOL_MISSING_TYPES @@ -42,18 +43,6 @@ static const int NCPUS_START = sizeof(unsigned long) * CHAR_BIT; // Linux >= 2.6.13 #define PSUTIL_HAVE_IOPRIO defined(__NR_ioprio_get) && defined(__NR_ioprio_set) -// Linux >= 2.6.36 (supposedly) and glibc >= 13 -#define PSUTIL_HAVE_PRLIMIT \ - (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)) && \ - (__GLIBC__ >= 2 && __GLIBC_MINOR__ >= 13) && \ - defined(__NR_prlimit64) - -#if PSUTIL_HAVE_PRLIMIT - #define _FILE_OFFSET_BITS 64 - #include <time.h> - #include <sys/resource.h> -#endif - // Should exist starting from CentOS 6 (year 2011). #ifdef CPU_ALLOC #define PSUTIL_HAVE_CPU_AFFINITY @@ -134,68 +123,6 @@ psutil_proc_ioprio_set(PyObject *self, PyObject *args) { #endif -#if PSUTIL_HAVE_PRLIMIT -/* - * A wrapper around prlimit(2); sets process resource limits. - * This can be used for both get and set, in which case extra - * 'soft' and 'hard' args must be provided. - */ -static PyObject * -psutil_linux_prlimit(PyObject *self, PyObject *args) { - pid_t pid; - int ret, resource; - struct rlimit old, new; - struct rlimit *newp = NULL; - PyObject *py_soft = NULL; - PyObject *py_hard = NULL; - - if (! PyArg_ParseTuple(args, _Py_PARSE_PID "i|OO", &pid, &resource, - &py_soft, &py_hard)) { - return NULL; - } - - // get - if (py_soft == NULL && py_hard == NULL) { - ret = prlimit(pid, resource, NULL, &old); - if (ret == -1) - return PyErr_SetFromErrno(PyExc_OSError); -#if defined(PSUTIL_HAVE_LONG_LONG) - if (sizeof(old.rlim_cur) > sizeof(long)) { - return Py_BuildValue("LL", - (PY_LONG_LONG)old.rlim_cur, - (PY_LONG_LONG)old.rlim_max); - } -#endif - return Py_BuildValue("ll", (long)old.rlim_cur, (long)old.rlim_max); - } - - // set - else { -#if defined(PSUTIL_HAVE_LARGEFILE_SUPPORT) - new.rlim_cur = PyLong_AsLongLong(py_soft); - if (new.rlim_cur == (rlim_t) - 1 && PyErr_Occurred()) - return NULL; - new.rlim_max = PyLong_AsLongLong(py_hard); - if (new.rlim_max == (rlim_t) - 1 && PyErr_Occurred()) - return NULL; -#else - new.rlim_cur = PyLong_AsLong(py_soft); - if (new.rlim_cur == (rlim_t) - 1 && PyErr_Occurred()) - return NULL; - new.rlim_max = PyLong_AsLong(py_hard); - if (new.rlim_max == (rlim_t) - 1 && PyErr_Occurred()) - return NULL; -#endif - newp = &new; - ret = prlimit(pid, resource, newp, &old); - if (ret == -1) - return PyErr_SetFromErrno(PyExc_OSError); - Py_RETURN_NONE; - } -} -#endif - - /* * Return disk mounted partitions as a list of tuples including device, * mount point and filesystem type @@ -501,7 +428,7 @@ psutil_net_if_duplex_speed(PyObject* self, PyObject* args) { sock = socket(AF_INET, SOCK_DGRAM, 0); if (sock == -1) return PyErr_SetFromOSErrnoWithSyscall("socket()"); - strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + PSUTIL_STRNCPY(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); // duplex and speed memset(ðcmd, 0, sizeof ethcmd); @@ -548,7 +475,7 @@ error: static PyMethodDef mod_methods[] = { // --- per-process functions -#ifdef PSUTIL_HAVE_IOPRIO +#if PSUTIL_HAVE_IOPRIO {"proc_ioprio_get", psutil_proc_ioprio_get, METH_VARARGS, "Get process I/O priority"}, {"proc_ioprio_set", psutil_proc_ioprio_set, METH_VARARGS, @@ -575,13 +502,9 @@ static PyMethodDef mod_methods[] = { {"linux_sysinfo", psutil_linux_sysinfo, METH_VARARGS, "A wrapper around sysinfo(), return system memory usage statistics"}, -#if PSUTIL_HAVE_PRLIMIT - {"linux_prlimit", psutil_linux_prlimit, METH_VARARGS, - "Get or set process resource limits."}, -#endif // --- others - {"set_testing", psutil_set_testing, METH_NOARGS, - "Set psutil in testing mode"}, + {"set_debug", psutil_set_debug, METH_VARARGS, + "Enable or disable PSUTIL_DEBUG messages"}, {NULL, NULL, 0, NULL} }; @@ -609,7 +532,6 @@ static PyMethodDef mod_methods[] = { void init_psutil_linux(void) #endif /* PY_MAJOR_VERSION */ { - PyObject *v; #if PY_MAJOR_VERSION >= 3 PyObject *mod = PyModule_Create(&moduledef); #else @@ -619,52 +541,12 @@ static PyMethodDef mod_methods[] = { INITERR; if (PyModule_AddIntConstant(mod, "version", PSUTIL_VERSION)) INITERR; -#if PSUTIL_HAVE_PRLIMIT - if (PyModule_AddIntConstant(mod, "RLIMIT_AS", RLIMIT_AS)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_CORE", RLIMIT_CORE)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_CPU", RLIMIT_CPU)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_DATA", RLIMIT_DATA)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_FSIZE", RLIMIT_FSIZE)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_LOCKS", RLIMIT_LOCKS)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_MEMLOCK", RLIMIT_MEMLOCK)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_NOFILE", RLIMIT_NOFILE)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_NPROC", RLIMIT_NPROC)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_RSS", RLIMIT_RSS)) INITERR; - if (PyModule_AddIntConstant(mod, "RLIMIT_STACK", RLIMIT_STACK)) INITERR; - -#if defined(HAVE_LONG_LONG) - if (sizeof(RLIM_INFINITY) > sizeof(long)) { - v = PyLong_FromLongLong((PY_LONG_LONG) RLIM_INFINITY); - } else -#endif - { - v = PyLong_FromLong((long) RLIM_INFINITY); - } - if (v) { - PyModule_AddObject(mod, "RLIM_INFINITY", v); - } - -#ifdef RLIMIT_MSGQUEUE - if (PyModule_AddIntConstant(mod, "RLIMIT_MSGQUEUE", RLIMIT_MSGQUEUE)) INITERR; -#endif -#ifdef RLIMIT_NICE - if (PyModule_AddIntConstant(mod, "RLIMIT_NICE", RLIMIT_NICE)) INITERR; -#endif -#ifdef RLIMIT_RTPRIO - if (PyModule_AddIntConstant(mod, "RLIMIT_RTPRIO", RLIMIT_RTPRIO)) INITERR; -#endif -#ifdef RLIMIT_RTTIME - if (PyModule_AddIntConstant(mod, "RLIMIT_RTTIME", RLIMIT_RTTIME)) INITERR; -#endif -#ifdef RLIMIT_SIGPENDING - if (PyModule_AddIntConstant(mod, "RLIMIT_SIGPENDING", RLIMIT_SIGPENDING)) - INITERR; -#endif -#endif if (PyModule_AddIntConstant(mod, "DUPLEX_HALF", DUPLEX_HALF)) INITERR; if (PyModule_AddIntConstant(mod, "DUPLEX_FULL", DUPLEX_FULL)) INITERR; if (PyModule_AddIntConstant(mod, "DUPLEX_UNKNOWN", DUPLEX_UNKNOWN)) INITERR; + psutil_setup(); + if (mod == NULL) INITERR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index c51c1c78..8c5990ae 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. * @@ -14,20 +14,14 @@ #include <stdio.h> #include <utmpx.h> #include <sys/sysctl.h> -#include <sys/vmmeter.h> #include <libproc.h> #include <sys/proc_info.h> #include <netinet/tcp_fsm.h> #include <arpa/inet.h> #include <net/if_dl.h> #include <pwd.h> - +#include <unistd.h> #include <mach/mach.h> -#include <mach/task.h> -#include <mach/mach_init.h> -#include <mach/host_info.h> -#include <mach/mach_host.h> -#include <mach/mach_traps.h> #include <mach/mach_vm.h> #include <mach/shared_region.h> @@ -44,6 +38,7 @@ #include "_psutil_common.h" #include "_psutil_posix.h" #include "arch/osx/process_info.h" +#include "arch/osx/cpu.h" #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) @@ -98,7 +93,8 @@ psutil_task_for_pid(pid_t pid, mach_port_t *task) if (psutil_pid_exists(pid) == 0) NoSuchProcess("task_for_pid"); else if (psutil_is_zombie(pid) == 1) - PyErr_SetString(ZombieProcessError, "task_for_pid() failed"); + PyErr_SetString(ZombieProcessError, + "task_for_pid -> psutil_is_zombie -> 1"); else { psutil_debug( "task_for_pid() failed (pid=%ld, err=%i, errno=%i, msg='%s'); " @@ -113,6 +109,72 @@ psutil_task_for_pid(pid_t pid, mach_port_t *task) /* + * A wrapper around proc_pidinfo(PROC_PIDLISTFDS), which dynamically sets + * the buffer size. + */ +static struct proc_fdinfo* +psutil_proc_list_fds(pid_t pid, int *num_fds) { + int ret; + int fds_size = 0; + int max_size = 24 * 1024 * 1024; // 24M + struct proc_fdinfo *fds_pointer = NULL; + + errno = 0; + ret = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, NULL, 0); + if (ret <= 0) { + psutil_raise_for_pid(pid, "proc_pidinfo(PROC_PIDLISTFDS) 1/2"); + goto error; + } + + while (1) { + if (ret > fds_size) { + while (ret > fds_size) { + fds_size += PROC_PIDLISTFD_SIZE * 32; + if (fds_size > max_size) { + PyErr_Format(PyExc_RuntimeError, + "prevent malloc() to allocate > 24M"); + goto error; + } + } + + if (fds_pointer != NULL) { + free(fds_pointer); + } + fds_pointer = malloc(fds_size); + + if (fds_pointer == NULL) { + PyErr_NoMemory(); + goto error; + } + } + + errno = 0; + ret = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, fds_pointer, fds_size); + if (ret <= 0) { + psutil_raise_for_pid(pid, "proc_pidinfo(PROC_PIDLISTFDS) 2/2"); + goto error; + } + + if (ret + (int)PROC_PIDLISTFD_SIZE >= fds_size) { + psutil_debug("PROC_PIDLISTFDS: make room for 1 extra fd"); + ret = fds_size + (int)PROC_PIDLISTFD_SIZE; + continue; + } + + break; + } + + *num_fds = (ret / (int)PROC_PIDLISTFD_SIZE); + return fds_pointer; + +error: + if (fds_pointer != NULL) + free(fds_pointer); + return NULL; +} + + +/* * Return a Python list of all the PIDs running on the system. */ static PyObject * @@ -297,11 +359,21 @@ psutil_proc_exe(PyObject *self, PyObject *args) { errno = 0; ret = proc_pidpath(pid, &buf, sizeof(buf)); if (ret == 0) { - if (pid == 0) + if (pid == 0) { AccessDenied("automatically set for PID 0"); - else + return NULL; + } + else if (errno == ENOENT) { + // It may happen (file not found error) if the process is + // still alive but the executable which launched it got + // deleted, see: + // https://github.com/giampaolo/psutil/issues/1738 + return Py_BuildValue("s", ""); + } + else { psutil_raise_for_pid(pid, "proc_pidpath()"); - return NULL; + return NULL; + } } return PyUnicode_DecodeFSDefault(buf); } @@ -342,50 +414,6 @@ psutil_proc_environ(PyObject *self, PyObject *args) { /* - * Return the number of logical CPUs in the system. - * XXX this could be shared with BSD. - */ -static PyObject * -psutil_cpu_count_logical(PyObject *self, PyObject *args) { - /* - int mib[2]; - int ncpu; - size_t len; - mib[0] = CTL_HW; - mib[1] = HW_NCPU; - len = sizeof(ncpu); - - if (sysctl(mib, 2, &ncpu, &len, NULL, 0) == -1) - Py_RETURN_NONE; // mimic os.cpu_count() - else - return Py_BuildValue("i", ncpu); - */ - int num; - size_t size = sizeof(int); - - if (sysctlbyname("hw.logicalcpu", &num, &size, NULL, 2)) - Py_RETURN_NONE; // mimic os.cpu_count() - else - return Py_BuildValue("i", num); -} - - -/* - * Return the number of physical CPUs in the system. - */ -static PyObject * -psutil_cpu_count_phys(PyObject *self, PyObject *args) { - int num; - size_t size = sizeof(int); - - if (sysctlbyname("hw.physicalcpu", &num, &size, NULL, 0)) - Py_RETURN_NONE; // mimic os.cpu_count() - else - return Py_BuildValue("i", num); -} - - -/* * Indicates if the given virtual address on the given architecture is in the * shared VM region. */ @@ -429,7 +457,7 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { mach_vm_size_t size = 0; mach_msg_type_number_t info_count = VM_REGION_TOP_INFO_COUNT; kern_return_t kr; - vm_size_t page_size; + long pagesize = psutil_getpagesize(); mach_vm_address_t addr = MACH_VM_MIN_ADDRESS; mach_port_t task = MACH_PORT_NULL; vm_region_top_info_data_t info; @@ -493,11 +521,7 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { } mach_port_deallocate(mach_task_self(), task); - - if (host_page_size(mach_host_self(), &page_size) != KERN_SUCCESS) - page_size = PAGE_SIZE; - - return Py_BuildValue("K", private_pages * page_size); + return Py_BuildValue("K", private_pages * pagesize); } @@ -513,7 +537,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { uint64_t total; size_t len = sizeof(total); vm_statistics_data_t vm; - int pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); // physical mem mib[0] = CTL_HW; mib[1] = HW_MEMSIZE; @@ -553,7 +577,7 @@ psutil_swap_mem(PyObject *self, PyObject *args) { size_t size; struct xsw_usage totals; vm_statistics_data_t vmstat; - int pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); mib[0] = CTL_VM; mib[1] = VM_SWAPUSAGE; @@ -580,36 +604,6 @@ psutil_swap_mem(PyObject *self, PyObject *args) { /* - * Return a Python tuple representing user, kernel and idle CPU times - */ -static PyObject * -psutil_cpu_times(PyObject *self, PyObject *args) { - mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT; - kern_return_t error; - host_cpu_load_info_data_t r_load; - - mach_port_t host_port = mach_host_self(); - error = host_statistics(host_port, HOST_CPU_LOAD_INFO, - (host_info_t)&r_load, &count); - if (error != KERN_SUCCESS) { - return PyErr_Format( - PyExc_RuntimeError, - "host_statistics(HOST_CPU_LOAD_INFO) syscall failed: %s", - mach_error_string(error)); - } - mach_port_deallocate(mach_task_self(), host_port); - - return Py_BuildValue( - "(dddd)", - (double)r_load.cpu_ticks[CPU_STATE_USER] / CLK_TCK, - (double)r_load.cpu_ticks[CPU_STATE_NICE] / CLK_TCK, - (double)r_load.cpu_ticks[CPU_STATE_SYSTEM] / CLK_TCK, - (double)r_load.cpu_ticks[CPU_STATE_IDLE] / CLK_TCK - ); -} - - -/* * Return a Python list of tuple representing per-cpu times */ static PyObject * @@ -676,37 +670,6 @@ error: /* - * Retrieve CPU frequency. - */ -static PyObject * -psutil_cpu_freq(PyObject *self, PyObject *args) { - int64_t curr; - int64_t min; - int64_t max; - size_t size = sizeof(int64_t); - - if (sysctlbyname("hw.cpufrequency", &curr, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('hw.cpufrequency')"); - } - if (sysctlbyname("hw.cpufrequency_min", &min, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('hw.cpufrequency_min')"); - } - if (sysctlbyname("hw.cpufrequency_max", &max, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('hw.cpufrequency_max')"); - } - - return Py_BuildValue( - "KKK", - curr / 1000 / 1000, - min / 1000 / 1000, - max / 1000 / 1000); -} - - -/* * Return a Python float indicating the system boot time expressed in * seconds since the epoch. */ @@ -894,7 +857,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (err != KERN_SUCCESS) { // errcode 4 is "invalid argument" (access denied) if (err == 4) { - AccessDenied("task_info"); + AccessDenied("task_info(TASK_BASIC_INFO)"); } else { // otherwise throw a runtime error with appropriate error code @@ -969,15 +932,12 @@ error: static PyObject * psutil_proc_open_files(PyObject *self, PyObject *args) { pid_t pid; - int pidinfo_result; - int iterations; + int num_fds; int i; unsigned long nb; - struct proc_fdinfo *fds_pointer = NULL; struct proc_fdinfo *fdp_pointer; struct vnode_fdinfowithpath vi; - PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_path = NULL; @@ -988,23 +948,11 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) goto error; - pidinfo_result = psutil_proc_pidinfo(pid, PROC_PIDLISTFDS, 0, NULL, 0); - if (pidinfo_result <= 0) - goto error; - - fds_pointer = malloc(pidinfo_result); - if (fds_pointer == NULL) { - PyErr_NoMemory(); - goto error; - } - pidinfo_result = psutil_proc_pidinfo( - pid, PROC_PIDLISTFDS, 0, fds_pointer, pidinfo_result); - if (pidinfo_result <= 0) + fds_pointer = psutil_proc_list_fds(pid, &num_fds); + if (fds_pointer == NULL) goto error; - iterations = (pidinfo_result / PROC_PIDLISTFD_SIZE); - - for (i = 0; i < iterations; i++) { + for (i = 0; i < num_fds; i++) { fdp_pointer = &fds_pointer[i]; if (fdp_pointer->proc_fdtype == PROX_FDTYPE_VNODE) { @@ -1071,15 +1019,12 @@ error: static PyObject * psutil_proc_connections(PyObject *self, PyObject *args) { pid_t pid; - int pidinfo_result; - int iterations; + int num_fds; int i; unsigned long nb; - struct proc_fdinfo *fds_pointer = NULL; struct proc_fdinfo *fdp_pointer; struct socket_fdinfo si; - PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_laddr = NULL; @@ -1100,25 +1045,11 @@ psutil_proc_connections(PyObject *self, PyObject *args) { goto error; } - if (pid == 0) - return py_retlist; - pidinfo_result = psutil_proc_pidinfo(pid, PROC_PIDLISTFDS, 0, NULL, 0); - if (pidinfo_result <= 0) - goto error; - - fds_pointer = malloc(pidinfo_result); - if (fds_pointer == NULL) { - PyErr_NoMemory(); - goto error; - } - - pidinfo_result = psutil_proc_pidinfo( - pid, PROC_PIDLISTFDS, 0, fds_pointer, pidinfo_result); - if (pidinfo_result <= 0) + fds_pointer = psutil_proc_list_fds(pid, &num_fds); + if (fds_pointer == NULL) goto error; - iterations = (pidinfo_result / PROC_PIDLISTFD_SIZE); - for (i = 0; i < iterations; i++) { + for (i = 0; i < num_fds; i++) { py_tuple = NULL; py_laddr = NULL; py_raddr = NULL; @@ -1130,9 +1061,18 @@ psutil_proc_connections(PyObject *self, PyObject *args) { PROC_PIDFDSOCKETINFO, &si, sizeof(si)); // --- errors checking - if ((nb <= 0) || (nb < sizeof(si))) { + if ((nb <= 0) || (nb < sizeof(si)) || (errno != 0)) { if (errno == EBADF) { // let's assume socket has been closed + psutil_debug("proc_pidfdinfo(PROC_PIDFDSOCKETINFO) -> " + "EBADF (ignored)"); + continue; + } + else if (errno == EOPNOTSUPP) { + // may happen sometimes, see: + // https://github.com/giampaolo/psutil/issues/1512 + psutil_debug("proc_pidfdinfo(PROC_PIDFDSOCKETINFO) -> " + "EOPNOTSUPP (ignored)"); continue; } else { @@ -1166,11 +1106,6 @@ psutil_proc_connections(PyObject *self, PyObject *args) { if (inseq == 0) continue; - if (errno != 0) { - PyErr_SetFromErrno(PyExc_OSError); - goto error; - } - if ((family == AF_INET) || (family == AF_INET6)) { if (family == AF_INET) { inet_ntop(AF_INET, @@ -1275,30 +1210,18 @@ error: static PyObject * psutil_proc_num_fds(PyObject *self, PyObject *args) { pid_t pid; - int pidinfo_result; - int num; + int num_fds; struct proc_fdinfo *fds_pointer; if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) return NULL; - pidinfo_result = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, NULL, 0); - if (pidinfo_result <= 0) - return PyErr_SetFromErrno(PyExc_OSError); - - fds_pointer = malloc(pidinfo_result); + fds_pointer = psutil_proc_list_fds(pid, &num_fds); if (fds_pointer == NULL) - return PyErr_NoMemory(); - pidinfo_result = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, fds_pointer, - pidinfo_result); - if (pidinfo_result <= 0) { - free(fds_pointer); - return PyErr_SetFromErrno(PyExc_OSError); - } + return NULL; - num = (pidinfo_result / PROC_PIDLISTFD_SIZE); free(fds_pointer); - return Py_BuildValue("i", num); + return Py_BuildValue("i", num_fds); } @@ -1624,37 +1547,6 @@ error: /* - * Return CPU statistics. - */ -static PyObject * -psutil_cpu_stats(PyObject *self, PyObject *args) { - struct vmmeter vmstat; - kern_return_t ret; - mach_msg_type_number_t count = sizeof(vmstat) / sizeof(integer_t); - mach_port_t mport = mach_host_self(); - - ret = host_statistics(mport, HOST_VM_INFO, (host_info_t)&vmstat, &count); - if (ret != KERN_SUCCESS) { - PyErr_Format( - PyExc_RuntimeError, - "host_statistics(HOST_VM_INFO) failed: %s", - mach_error_string(ret)); - return NULL; - } - mach_port_deallocate(mach_task_self(), mport); - - return Py_BuildValue( - "IIIII", - vmstat.v_swtch, // ctx switches - vmstat.v_intr, // interrupts - vmstat.v_soft, // software interrupts - vmstat.v_syscall, // syscalls - vmstat.v_trap // traps - ); -} - - -/* * Return battery information. */ static PyObject * @@ -1778,8 +1670,8 @@ static PyMethodDef mod_methods[] = { "Returns a list of PIDs currently running on the system"}, {"cpu_count_logical", psutil_cpu_count_logical, METH_VARARGS, "Return number of logical CPUs on the system"}, - {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, - "Return number of physical CPUs on the system"}, + {"cpu_count_cores", psutil_cpu_count_cores, METH_VARARGS, + "Return number of CPU cores on the system"}, {"virtual_mem", psutil_virtual_mem, METH_VARARGS, "Return system virtual memory stats"}, {"swap_mem", psutil_swap_mem, METH_VARARGS, @@ -1807,8 +1699,8 @@ static PyMethodDef mod_methods[] = { "Return battery information."}, // --- others - {"set_testing", psutil_set_testing, METH_NOARGS, - "Set psutil in testing mode"}, + {"set_debug", psutil_set_debug, METH_VARARGS, + "Enable or disable PSUTIL_DEBUG messages"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index fa554be9..7c0d548c 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -15,6 +15,7 @@ #include <sys/socket.h> #include <sys/ioctl.h> #include <net/if.h> +#include <unistd.h> #ifdef PSUTIL_SUNOS10 #include "arch/solaris/v10/ifaddrs.h" @@ -28,22 +29,60 @@ #include <netdb.h> #include <linux/types.h> #include <linux/if_packet.h> -#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) +#endif +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) #include <netdb.h> #include <netinet/in.h> #include <net/if_dl.h> #include <sys/sockio.h> #include <net/if_media.h> #include <net/if.h> -#elif defined(PSUTIL_SUNOS) +#endif +#if defined(PSUTIL_SUNOS) #include <netdb.h> #include <sys/sockio.h> -#elif defined(PSUTIL_AIX) +#endif +#if defined(PSUTIL_AIX) #include <netdb.h> #endif +#if defined(PSUTIL_LINUX) || defined(PSUTIL_FREEBSD) + #include <sys/resource.h> +#endif #include "_psutil_common.h" + +// ==================================================================== +// --- Utils +// ==================================================================== + + +/* + * From "man getpagesize" on Linux, https://linux.die.net/man/2/getpagesize: + * + * > In SUSv2 the getpagesize() call is labeled LEGACY, and in POSIX.1-2001 + * > it has been dropped. + * > Portable applications should employ sysconf(_SC_PAGESIZE) instead + * > of getpagesize(). + * > Most systems allow the synonym _SC_PAGE_SIZE for _SC_PAGESIZE. + * > Whether getpagesize() is present as a Linux system call depends on the + * > architecture. + */ +long +psutil_getpagesize(void) { +#ifdef _SC_PAGESIZE + // recommended POSIX + return sysconf(_SC_PAGESIZE); +#elif _SC_PAGE_SIZE + // alias + return sysconf(_SC_PAGE_SIZE); +#else + // legacy + return (long) getpagesize(); +#endif +} + + /* * Check if PID exists. Return values: * 1: exists @@ -108,7 +147,7 @@ psutil_pid_exists(pid_t pid) { */ void psutil_raise_for_pid(long pid, char *syscall) { - if (errno != 0) // unlikely + if (errno != 0) PyErr_SetFromOSErrnoWithSyscall(syscall); else if (psutil_pid_exists(pid) == 0) NoSuchProcess(syscall); @@ -117,6 +156,18 @@ psutil_raise_for_pid(long pid, char *syscall) { } +// ==================================================================== +// --- Python wrappers +// ==================================================================== + + +// Exposed so we can test it against Python's stdlib. +static PyObject * +psutil_getpagesize_pywrapper(PyObject *self, PyObject *args) { + return Py_BuildValue("l", psutil_getpagesize()); +} + + /* * Given a PID return process priority as a Python integer. */ @@ -192,8 +243,8 @@ psutil_convert_ipaddr(struct sockaddr *addr, int family) { // XXX we get here on FreeBSD when processing 'lo' / AF_INET6 // broadcast. Not sure what to do other than returning None. // ifconfig does not show anything BTW. - //PyErr_Format(PyExc_RuntimeError, gai_strerror(err)); - //return NULL; + // PyErr_Format(PyExc_RuntimeError, gai_strerror(err)); + // return NULL; Py_INCREF(Py_None); return Py_None; } @@ -356,10 +407,10 @@ psutil_net_if_mtu(PyObject *self, PyObject *args) { goto error; #ifdef PSUTIL_SUNOS10 - strncpy(lifr.lifr_name, nic_name, sizeof(lifr.lifr_name)); + PSUTIL_STRNCPY(lifr.lifr_name, nic_name, sizeof(lifr.lifr_name)); ret = ioctl(sock, SIOCGIFMTU, &lifr); #else - strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + PSUTIL_STRNCPY(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); ret = ioctl(sock, SIOCGIFMTU, &ifr); #endif if (ret == -1) @@ -385,7 +436,7 @@ error: * http://www.i-scream.org/libstatgrab/ */ static PyObject * -psutil_net_if_flags(PyObject *self, PyObject *args) { +psutil_net_if_is_running(PyObject *self, PyObject *args) { char *nic_name; int sock = -1; int ret; @@ -398,13 +449,13 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { if (sock == -1) goto error; - strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + PSUTIL_STRNCPY(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); ret = ioctl(sock, SIOCGIFFLAGS, &ifr); if (ret == -1) goto error; close(sock); - if ((ifr.ifr_flags & IFF_UP) != 0) + if ((ifr.ifr_flags & IFF_RUNNING) != 0) return Py_BuildValue("O", Py_True); else return Py_BuildValue("O", Py_False); @@ -578,7 +629,7 @@ psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { sock = socket(AF_INET, SOCK_DGRAM, 0); if (sock == -1) return PyErr_SetFromErrno(PyExc_OSError); - strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + PSUTIL_STRNCPY(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); // speed / duplex memset(&ifmed, 0, sizeof(struct ifmediareq)); @@ -621,8 +672,10 @@ static PyMethodDef mod_methods[] = { "Retrieve NICs information"}, {"net_if_mtu", psutil_net_if_mtu, METH_VARARGS, "Retrieve NIC MTU"}, - {"net_if_flags", psutil_net_if_flags, METH_VARARGS, - "Retrieve NIC flags"}, + {"net_if_is_running", psutil_net_if_is_running, METH_VARARGS, + "Return True if the NIC is running."}, + {"getpagesize", psutil_getpagesize_pywrapper, METH_VARARGS, + "Return memory page size."}, #if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) {"net_if_duplex_speed", psutil_net_if_duplex_speed, METH_VARARGS, "Return NIC stats."}, @@ -668,6 +721,102 @@ static PyMethodDef mod_methods[] = { if (PyModule_AddIntConstant(mod, "AF_LINK", AF_LINK)) INITERR; #endif +#if defined(PSUTIL_LINUX) || defined(PSUTIL_FREEBSD) + PyObject *v; + +#ifdef RLIMIT_AS + if (PyModule_AddIntConstant(mod, "RLIMIT_AS", RLIMIT_AS)) INITERR; +#endif + +#ifdef RLIMIT_CORE + if (PyModule_AddIntConstant(mod, "RLIMIT_CORE", RLIMIT_CORE)) INITERR; +#endif + +#ifdef RLIMIT_CPU + if (PyModule_AddIntConstant(mod, "RLIMIT_CPU", RLIMIT_CPU)) INITERR; +#endif + +#ifdef RLIMIT_DATA + if (PyModule_AddIntConstant(mod, "RLIMIT_DATA", RLIMIT_DATA)) INITERR; +#endif + +#ifdef RLIMIT_FSIZE + if (PyModule_AddIntConstant(mod, "RLIMIT_FSIZE", RLIMIT_FSIZE)) INITERR; +#endif + +#ifdef RLIMIT_MEMLOCK + if (PyModule_AddIntConstant(mod, "RLIMIT_MEMLOCK", RLIMIT_MEMLOCK)) INITERR; +#endif + +#ifdef RLIMIT_NOFILE + if (PyModule_AddIntConstant(mod, "RLIMIT_NOFILE", RLIMIT_NOFILE)) INITERR; +#endif + +#ifdef RLIMIT_NPROC + if (PyModule_AddIntConstant(mod, "RLIMIT_NPROC", RLIMIT_NPROC)) INITERR; +#endif + +#ifdef RLIMIT_RSS + if (PyModule_AddIntConstant(mod, "RLIMIT_RSS", RLIMIT_RSS)) INITERR; +#endif + +#ifdef RLIMIT_STACK + if (PyModule_AddIntConstant(mod, "RLIMIT_STACK", RLIMIT_STACK)) INITERR; +#endif + +// Linux specific + +#ifdef RLIMIT_LOCKS + if (PyModule_AddIntConstant(mod, "RLIMIT_LOCKS", RLIMIT_LOCKS)) INITERR; +#endif + +#ifdef RLIMIT_MSGQUEUE + if (PyModule_AddIntConstant(mod, "RLIMIT_MSGQUEUE", RLIMIT_MSGQUEUE)) INITERR; +#endif + +#ifdef RLIMIT_NICE + if (PyModule_AddIntConstant(mod, "RLIMIT_NICE", RLIMIT_NICE)) INITERR; +#endif + +#ifdef RLIMIT_RTPRIO + if (PyModule_AddIntConstant(mod, "RLIMIT_RTPRIO", RLIMIT_RTPRIO)) INITERR; +#endif + +#ifdef RLIMIT_RTTIME + if (PyModule_AddIntConstant(mod, "RLIMIT_RTTIME", RLIMIT_RTTIME)) INITERR; +#endif + +#ifdef RLIMIT_SIGPENDING + if (PyModule_AddIntConstant(mod, "RLIMIT_SIGPENDING", RLIMIT_SIGPENDING)) INITERR; +#endif + +// Free specific + +#ifdef RLIMIT_SWAP + if (PyModule_AddIntConstant(mod, "RLIMIT_SWAP", RLIMIT_SWAP)) INITERR; +#endif + +#ifdef RLIMIT_SBSIZE + if (PyModule_AddIntConstant(mod, "RLIMIT_SBSIZE", RLIMIT_SBSIZE)) INITERR; +#endif + +#ifdef RLIMIT_NPTS + if (PyModule_AddIntConstant(mod, "RLIMIT_NPTS", RLIMIT_NPTS)) INITERR; +#endif + +#if defined(HAVE_LONG_LONG) + if (sizeof(RLIM_INFINITY) > sizeof(long)) { + v = PyLong_FromLongLong((PY_LONG_LONG) RLIM_INFINITY); + } else +#endif + { + v = PyLong_FromLong((long) RLIM_INFINITY); + } + if (v) { + PyModule_AddObject(mod, "RLIM_INFINITY", v); + } +#endif // defined(PSUTIL_LINUX) || defined(PSUTIL_FREEBSD) + if (mod == NULL) INITERR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_posix.h b/psutil/_psutil_posix.h index 59b9e532..5a37e48b 100644 --- a/psutil/_psutil_posix.h +++ b/psutil/_psutil_posix.h @@ -4,5 +4,6 @@ * found in the LICENSE file. */ +long psutil_getpagesize(void); int psutil_pid_exists(pid_t pid); void psutil_raise_for_pid(pid_t pid, char *msg); diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 8aa7eadd..2e0bd943 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -496,7 +496,7 @@ proc_io_counters(PyObject* self, PyObject* args) { info.pr_inblk, info.pr_oublk); } - */ +*/ /* @@ -1024,7 +1024,7 @@ psutil_net_io_counters(PyObject *self, PyObject *args) { goto next; // check if this is a network interface by sending a ioctl - strncpy(ifr.lifr_name, ksp->ks_name, sizeof(ifr.lifr_name)); + PSUTIL_STRNCPY(ifr.lifr_name, ksp->ks_name, sizeof(ifr.lifr_name)); ret = ioctl(sock, SIOCGLIFFLAGS, &ifr); if (ret == -1) goto next; @@ -1437,10 +1437,10 @@ psutil_boot_time(PyObject *self, PyObject *args) { /* - * Return the number of physical CPU cores on the system. + * Return the number of CPU cores on the system. */ static PyObject * -psutil_cpu_count_phys(PyObject *self, PyObject *args) { +psutil_cpu_count_cores(PyObject *self, PyObject *args) { kstat_ctl_t *kc; kstat_t *ksp; int ncpus = 0; @@ -1515,7 +1515,7 @@ psutil_net_if_stats(PyObject* self, PyObject* args) { if (strcmp(ksp->ks_class, "net") != 0) continue; - strncpy(ifr.lifr_name, ksp->ks_name, sizeof(ifr.lifr_name)); + PSUTIL_STRNCPY(ifr.lifr_name, ksp->ks_name, sizeof(ifr.lifr_name)); ret = ioctl(sock, SIOCGLIFFLAGS, &ifr); if (ret == -1) continue; // not a network interface @@ -1669,8 +1669,8 @@ PsutilMethods[] = { "Return a Python dict of tuples for network I/O statistics."}, {"boot_time", psutil_boot_time, METH_VARARGS, "Return system boot time in seconds since the EPOCH."}, - {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, - "Return the number of physical CPUs on the system."}, + {"cpu_count_cores", psutil_cpu_count_cores, METH_VARARGS, + "Return the number of CPU cores on the system."}, {"net_connections", psutil_net_connections, METH_VARARGS, "Return TCP and UDP syste-wide open connections."}, {"net_if_stats", psutil_net_if_stats, METH_VARARGS, @@ -1679,8 +1679,8 @@ PsutilMethods[] = { "Return CPU statistics"}, // --- others - {"set_testing", psutil_set_testing, METH_NOARGS, - "Set psutil in testing mode"}, + {"set_debug", psutil_set_debug, METH_VARARGS, + "Enable or disable PSUTIL_DEBUG messages"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index f63c6a35..3cc6dbbd 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -22,7 +22,6 @@ #include <Psapi.h> // memory_info(), memory_maps() #include <signal.h> #include <tlhelp32.h> // threads(), PROCESSENTRY32 -#include <wtsapi32.h> // users() // Link with Iphlpapi.lib #pragma comment(lib, "IPHLPAPI.lib") @@ -45,35 +44,6 @@ static PyObject *TimeoutAbandoned; /* - * Return the number of logical, active CPUs. Return 0 if undetermined. - * See discussion at: https://bugs.python.org/issue33166#msg314631 - */ -unsigned int -psutil_get_num_cpus(int fail_on_err) { - unsigned int ncpus = 0; - - // Minimum requirement: Windows 7 - if (GetActiveProcessorCount != NULL) { - ncpus = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); - if ((ncpus == 0) && (fail_on_err == 1)) { - PyErr_SetFromWindowsErr(0); - } - } - else { - psutil_debug("GetActiveProcessorCount() not available; " - "using GetSystemInfo()"); - ncpus = (unsigned int)PSUTIL_SYSTEM_INFO.dwNumberOfProcessors; - if ((ncpus <= 0) && (fail_on_err == 1)) { - PyErr_SetString( - PyExc_RuntimeError, - "GetSystemInfo() failed to retrieve CPU count"); - } - } - return ncpus; -} - - -/* * Return a Python float representing the system uptime expressed in seconds * since the epoch. */ @@ -160,22 +130,15 @@ psutil_proc_kill(PyObject *self, PyObject *args) { return AccessDenied("automatically set for PID 0"); hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pid); + hProcess = psutil_check_phandle(hProcess, pid, 0); if (hProcess == NULL) { - if (GetLastError() == ERROR_INVALID_PARAMETER) { - // see https://github.com/giampaolo/psutil/issues/24 - psutil_debug("OpenProcess -> ERROR_INVALID_PARAMETER turned " - "into NoSuchProcess"); - NoSuchProcess("OpenProcess"); - } - else { - PyErr_SetFromWindowsErr(0); - } return NULL; } if (! TerminateProcess(hProcess, SIGTERM)) { // ERROR_ACCESS_DENIED may happen if the process already died. See: // https://github.com/giampaolo/psutil/issues/1099 + // http://bugs.python.org/issue14252 if (GetLastError() != ERROR_ACCESS_DENIED) { PyErr_SetFromOSErrnoWithSyscall("TerminateProcess"); return NULL; @@ -212,7 +175,7 @@ psutil_proc_wait(PyObject *self, PyObject *args) { Py_RETURN_NONE; } else { - PyErr_SetFromWindowsErr(0); + PyErr_SetFromOSErrnoWithSyscall("OpenProcess"); return NULL; } } @@ -281,7 +244,7 @@ psutil_proc_times(PyObject *self, PyObject *args) { if (GetLastError() == ERROR_ACCESS_DENIED) { // usually means the process has died so we throw a NoSuchProcess // here - NoSuchProcess("GetProcessTimes"); + NoSuchProcess("GetProcessTimes -> ERROR_ACCESS_DENIED"); } else { PyErr_SetFromWindowsErr(0); @@ -333,7 +296,7 @@ psutil_proc_cmdline(PyObject *self, PyObject *args, PyObject *kwdict) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) - return NoSuchProcess("psutil_pid_is_running"); + return NoSuchProcess("psutil_pid_is_running -> 0"); if (pid_return == -1) return NULL; @@ -357,7 +320,7 @@ psutil_proc_environ(PyObject *self, PyObject *args) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) - return NoSuchProcess("psutil_pid_is_running"); + return NoSuchProcess("psutil_pid_is_running -> 0"); if (pid_return == -1) return NULL; @@ -375,8 +338,8 @@ static PyObject * psutil_proc_exe(PyObject *self, PyObject *args) { DWORD pid; NTSTATUS status; - PVOID buffer; - ULONG bufferSize = 0x100; + PVOID buffer = NULL; + ULONG bufferSize = 0x104 * 2; // WIN_MAX_PATH * sizeof(wchar_t) SYSTEM_PROCESS_ID_INFORMATION processIdInfo; PyObject *py_exe; @@ -384,11 +347,14 @@ psutil_proc_exe(PyObject *self, PyObject *args) { return NULL; if (pid == 0) - return AccessDenied("forced for PID 0"); + return AccessDenied("automatically set for PID 0"); buffer = MALLOC_ZERO(bufferSize); - if (! buffer) - return PyErr_NoMemory(); + if (! buffer) { + PyErr_NoMemory(); + return NULL; + } + processIdInfo.ProcessId = (HANDLE)(ULONG_PTR)pid; processIdInfo.ImageName.Length = 0; processIdInfo.ImageName.MaximumLength = (USHORT)bufferSize; @@ -400,12 +366,41 @@ psutil_proc_exe(PyObject *self, PyObject *args) { sizeof(SYSTEM_PROCESS_ID_INFORMATION), NULL); - if (status == STATUS_INFO_LENGTH_MISMATCH) { + if ((status == STATUS_INFO_LENGTH_MISMATCH) && + (processIdInfo.ImageName.MaximumLength <= bufferSize)) + { + // Required length was NOT stored in MaximumLength (WOW64 issue). + ULONG maxBufferSize = 0x7FFF * 2; // NTFS_MAX_PATH * sizeof(wchar_t) + do { + // Iteratively double the size of the buffer up to maxBufferSize + bufferSize *= 2; + FREE(buffer); + buffer = MALLOC_ZERO(bufferSize); + if (! buffer) { + PyErr_NoMemory(); + return NULL; + } + + processIdInfo.ImageName.MaximumLength = (USHORT)bufferSize; + processIdInfo.ImageName.Buffer = buffer; + + status = NtQuerySystemInformation( + SystemProcessIdInformation, + &processIdInfo, + sizeof(SYSTEM_PROCESS_ID_INFORMATION), + NULL); + } while ((status == STATUS_INFO_LENGTH_MISMATCH) && + (bufferSize <= maxBufferSize)); + } + else if (status == STATUS_INFO_LENGTH_MISMATCH) { // Required length is stored in MaximumLength. FREE(buffer); buffer = MALLOC_ZERO(processIdInfo.ImageName.MaximumLength); - if (! buffer) - return PyErr_NoMemory(); + if (! buffer) { + PyErr_NoMemory(); + return NULL; + } + processIdInfo.ImageName.Buffer = buffer; status = NtQuerySystemInformation( @@ -418,7 +413,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (! NT_SUCCESS(status)) { FREE(buffer); if (psutil_pid_is_running(pid) == 0) - NoSuchProcess("NtQuerySystemInformation"); + NoSuchProcess("psutil_pid_is_running -> 0"); else psutil_SetFromNTStatusErr(status, "NtQuerySystemInformation"); return NULL; @@ -537,10 +532,10 @@ psutil_GetProcWsetInformation( if (!NT_SUCCESS(status)) { if (status == STATUS_ACCESS_DENIED) { - AccessDenied("NtQueryVirtualMemory"); + AccessDenied("NtQueryVirtualMemory -> STATUS_ACCESS_DENIED"); } else if (psutil_pid_is_running(pid) == 0) { - NoSuchProcess("psutil_pid_is_running"); + NoSuchProcess("psutil_pid_is_running -> 0"); } else { PyErr_Clear(); @@ -645,7 +640,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) - return NoSuchProcess("psutil_pid_is_running"); + return NoSuchProcess("psutil_pid_is_running -> 0"); if (pid_return == -1) return NULL; @@ -704,13 +699,13 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (pid == 0) { // raise AD instead of returning 0 as procexp is able to // retrieve useful information somehow - AccessDenied("automatically set for PID 0"); + AccessDenied("forced for PID 0"); goto error; } pid_return = psutil_pid_is_running(pid); if (pid_return == 0) { - NoSuchProcess("psutil_pid_is_running"); + NoSuchProcess("psutil_pid_is_running -> 0"); goto error; } if (pid_return == -1) @@ -810,54 +805,34 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { } -/* - * Return process username as a "DOMAIN//USERNAME" string. - */ -static PyObject * -psutil_proc_username(PyObject *self, PyObject *args) { - DWORD pid; - HANDLE processHandle = NULL; - HANDLE tokenHandle = NULL; - PTOKEN_USER user = NULL; - ULONG bufferSize; - WCHAR *name = NULL; - WCHAR *domainName = NULL; - ULONG nameSize; - ULONG domainNameSize; - SID_NAME_USE nameUse; - PyObject *py_username = NULL; - PyObject *py_domain = NULL; - PyObject *py_tuple = NULL; - - if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) - return NULL; +static PTOKEN_USER +_psutil_user_token_from_pid(DWORD pid) { + HANDLE hProcess = NULL; + HANDLE hToken = NULL; + PTOKEN_USER userToken = NULL; + ULONG bufferSize = 0x100; - processHandle = psutil_handle_from_pid( - pid, PROCESS_QUERY_LIMITED_INFORMATION); - if (processHandle == NULL) + hProcess = psutil_handle_from_pid(pid, PROCESS_QUERY_LIMITED_INFORMATION); + if (hProcess == NULL) return NULL; - if (!OpenProcessToken(processHandle, TOKEN_QUERY, &tokenHandle)) { + if (!OpenProcessToken(hProcess, TOKEN_QUERY, &hToken)) { PyErr_SetFromOSErrnoWithSyscall("OpenProcessToken"); goto error; } - CloseHandle(processHandle); - processHandle = NULL; - // Get the user SID. - bufferSize = 0x100; while (1) { - user = malloc(bufferSize); - if (user == NULL) { + userToken = malloc(bufferSize); + if (userToken == NULL) { PyErr_NoMemory(); goto error; } - if (!GetTokenInformation(tokenHandle, TokenUser, user, bufferSize, + if (!GetTokenInformation(hToken, TokenUser, userToken, bufferSize, &bufferSize)) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - free(user); + free(userToken); continue; } else { @@ -868,15 +843,45 @@ psutil_proc_username(PyObject *self, PyObject *args) { break; } - CloseHandle(tokenHandle); - tokenHandle = NULL; + CloseHandle(hProcess); + CloseHandle(hToken); + return userToken; + +error: + if (hProcess != NULL) + CloseHandle(hProcess); + if (hToken != NULL) + CloseHandle(hToken); + return NULL; +} + + +/* + * Return process username as a "DOMAIN//USERNAME" string. + */ +static PyObject * +psutil_proc_username(PyObject *self, PyObject *args) { + DWORD pid; + PTOKEN_USER userToken = NULL; + WCHAR *userName = NULL; + WCHAR *domainName = NULL; + ULONG nameSize = 0x100; + ULONG domainNameSize = 0x100; + SID_NAME_USE nameUse; + PyObject *py_username = NULL; + PyObject *py_domain = NULL; + PyObject *py_tuple = NULL; + + if (! PyArg_ParseTuple(args, _Py_PARSE_PID, &pid)) + return NULL; + userToken = _psutil_user_token_from_pid(pid); + if (userToken == NULL) + return NULL; // resolve the SID to a name - nameSize = 0x100; - domainNameSize = 0x100; while (1) { - name = malloc(nameSize * sizeof(WCHAR)); - if (name == NULL) { + userName = malloc(nameSize * sizeof(WCHAR)); + if (userName == NULL) { PyErr_NoMemory(); goto error; } @@ -885,14 +890,27 @@ psutil_proc_username(PyObject *self, PyObject *args) { PyErr_NoMemory(); goto error; } - if (!LookupAccountSidW(NULL, user->User.Sid, name, &nameSize, + if (!LookupAccountSidW(NULL, userToken->User.Sid, userName, &nameSize, domainName, &domainNameSize, &nameUse)) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - free(name); + free(userName); free(domainName); continue; } + else if (GetLastError() == ERROR_NONE_MAPPED) { + // From MS doc: + // https://docs.microsoft.com/en-us/windows/win32/api/winbase/ + // nf-winbase-lookupaccountsida + // If the function cannot find an account name for the SID, + // GetLastError returns ERROR_NONE_MAPPED. This can occur if + // a network time-out prevents the function from finding the + // name. It also occurs for SIDs that have no corresponding + // account name, such as a logon SID that identifies a logon + // session. + AccessDenied("LookupAccountSidW -> ERROR_NONE_MAPPED"); + goto error; + } else { PyErr_SetFromOSErrnoWithSyscall("LookupAccountSidW"); goto error; @@ -904,7 +922,7 @@ psutil_proc_username(PyObject *self, PyObject *args) { py_domain = PyUnicode_FromWideChar(domainName, wcslen(domainName)); if (! py_domain) goto error; - py_username = PyUnicode_FromWideChar(name, wcslen(name)); + py_username = PyUnicode_FromWideChar(userName, wcslen(userName)); if (! py_username) goto error; py_tuple = Py_BuildValue("OO", py_domain, py_username); @@ -913,23 +931,18 @@ psutil_proc_username(PyObject *self, PyObject *args) { Py_DECREF(py_domain); Py_DECREF(py_username); - free(name); + free(userName); free(domainName); - free(user); - + free(userToken); return py_tuple; error: - if (processHandle != NULL) - CloseHandle(processHandle); - if (tokenHandle != NULL) - CloseHandle(tokenHandle); - if (name != NULL) - free(name); + if (userName != NULL) + free(userName); if (domainName != NULL) free(domainName); - if (user != NULL) - free(user); + if (userToken != NULL) + free(userToken); Py_XDECREF(py_domain); Py_XDECREF(py_username); Py_XDECREF(py_tuple); @@ -1186,17 +1199,17 @@ psutil_proc_is_suspended(PyObject *self, PyObject *args) { static PyObject * psutil_users(PyObject *self, PyObject *args) { HANDLE hServer = WTS_CURRENT_SERVER_HANDLE; - WCHAR *buffer_user = NULL; - LPTSTR buffer_addr = NULL; - PWTS_SESSION_INFO sessions = NULL; + LPWSTR buffer_user = NULL; + LPWSTR buffer_addr = NULL; + LPWSTR buffer_info = NULL; + PWTS_SESSION_INFOW sessions = NULL; DWORD count; DWORD i; DWORD sessionId; DWORD bytes; PWTS_CLIENT_ADDRESS address; char address_str[50]; - WINSTATION_INFO station_info; - ULONG returnLen; + PWTSINFOW wts_info; PyObject *py_tuple = NULL; PyObject *py_address = NULL; PyObject *py_username = NULL; @@ -1205,8 +1218,21 @@ psutil_users(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; - if (WTSEnumerateSessions(hServer, 0, 1, &sessions, &count) == 0) { - PyErr_SetFromOSErrnoWithSyscall("WTSEnumerateSessions"); + if (WTSEnumerateSessionsW == NULL || + WTSQuerySessionInformationW == NULL || + WTSFreeMemory == NULL) { + // If we don't run in an environment that is a Remote Desktop Services environment + // the Wtsapi32 proc might not be present. + // https://docs.microsoft.com/en-us/windows/win32/termserv/run-time-linking-to-wtsapi32-dll + return py_retlist; + } + + if (WTSEnumerateSessionsW(hServer, 0, 1, &sessions, &count) == 0) { + if (ERROR_CALL_NOT_IMPLEMENTED == GetLastError()) { + // On Windows Nano server, the Wtsapi32 API can be present, but return WinError 120. + return py_retlist; + } + PyErr_SetFromOSErrnoWithSyscall("WTSEnumerateSessionsW"); goto error; } @@ -1218,9 +1244,12 @@ psutil_users(PyObject *self, PyObject *args) { WTSFreeMemory(buffer_user); if (buffer_addr != NULL) WTSFreeMemory(buffer_addr); + if (buffer_info != NULL) + WTSFreeMemory(buffer_info); buffer_user = NULL; buffer_addr = NULL; + buffer_info = NULL; // username bytes = 0; @@ -1234,50 +1263,49 @@ psutil_users(PyObject *self, PyObject *args) { // address bytes = 0; - if (WTSQuerySessionInformation(hServer, sessionId, WTSClientAddress, - &buffer_addr, &bytes) == 0) { - PyErr_SetFromOSErrnoWithSyscall("WTSQuerySessionInformation"); + if (WTSQuerySessionInformationW(hServer, sessionId, WTSClientAddress, + &buffer_addr, &bytes) == 0) { + PyErr_SetFromOSErrnoWithSyscall("WTSQuerySessionInformationW"); goto error; } address = (PWTS_CLIENT_ADDRESS)buffer_addr; - if (address->AddressFamily == 0) { // AF_INET + if (address->AddressFamily == 2) { // AF_INET == 2 sprintf_s(address_str, _countof(address_str), "%u.%u.%u.%u", - address->Address[0], - address->Address[1], + // The IP address is offset by two bytes from the start of the Address member of the WTS_CLIENT_ADDRESS structure. address->Address[2], - address->Address[3]); + address->Address[3], + address->Address[4], + address->Address[5]); py_address = Py_BuildValue("s", address_str); if (!py_address) goto error; } else { + Py_INCREF(Py_None); py_address = Py_None; } // login time - if (! WinStationQueryInformationW( - hServer, - sessionId, - WinStationInformation, - &station_info, - sizeof(station_info), - &returnLen)) - { - PyErr_SetFromOSErrnoWithSyscall("WinStationQueryInformationW"); + bytes = 0; + if (WTSQuerySessionInformationW(hServer, sessionId, WTSSessionInfo, + &buffer_info, &bytes) == 0) { + PyErr_SetFromOSErrnoWithSyscall("WTSQuerySessionInformationW"); goto error; } + wts_info = (PWTSINFOW)buffer_info; py_username = PyUnicode_FromWideChar(buffer_user, wcslen(buffer_user)); if (py_username == NULL) goto error; + py_tuple = Py_BuildValue( "OOd", py_username, py_address, - psutil_FiletimeToUnixTime(station_info.ConnectTime) + psutil_LargeIntegerToUnixTime(wts_info->ConnectTime) ); if (!py_tuple) goto error; @@ -1291,6 +1319,7 @@ psutil_users(PyObject *self, PyObject *args) { WTSFreeMemory(sessions); WTSFreeMemory(buffer_user); WTSFreeMemory(buffer_addr); + WTSFreeMemory(buffer_info); return py_retlist; error: @@ -1305,6 +1334,8 @@ error: WTSFreeMemory(buffer_user); if (buffer_addr != NULL) WTSFreeMemory(buffer_addr); + if (buffer_info != NULL) + WTSFreeMemory(buffer_info); return NULL; } @@ -1584,8 +1615,8 @@ PsutilMethods[] = { "Determine if the process exists in the current process list."}, {"cpu_count_logical", psutil_cpu_count_logical, METH_VARARGS, "Returns the number of logical CPUs on the system"}, - {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, - "Returns the number of physical CPUs on the system"}, + {"cpu_count_cores", psutil_cpu_count_cores, METH_VARARGS, + "Returns the number of CPU cores on the system"}, {"boot_time", psutil_boot_time, METH_VARARGS, "Return the system boot time expressed in seconds since the epoch."}, {"virtual_mem", psutil_virtual_mem, METH_VARARGS, @@ -1641,12 +1672,12 @@ PsutilMethods[] = { "Stop a service"}, // --- windows API bindings - {"win32_QueryDosDevice", psutil_win32_QueryDosDevice, METH_VARARGS, + {"QueryDosDevice", psutil_QueryDosDevice, METH_VARARGS, "QueryDosDevice binding"}, // --- others - {"set_testing", psutil_set_testing, METH_NOARGS, - "Set psutil in testing mode"}, + {"set_debug", psutil_set_debug, METH_VARARGS, + "Enable or disable PSUTIL_DEBUG messages"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 0446656c..52e73eee 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -30,8 +30,8 @@ from ._common import usage_percent from ._compat import long from ._compat import lru_cache from ._compat import PY3 +from ._compat import range from ._compat import unicode -from ._compat import xrange from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS from ._psutil_windows import HIGH_PRIORITY_CLASS @@ -203,7 +203,7 @@ def convert_dos_path(s): if s.startswith("\\??\\"): return s[4:] rawdrive = '\\'.join(s.split('\\')[:3]) - driveletter = cext.win32_QueryDosDevice(rawdrive) + driveletter = cext.QueryDosDevice(rawdrive) remainder = s[len(rawdrive):] return os.path.join(driveletter, remainder) @@ -247,8 +247,16 @@ def virtual_memory(): def swap_memory(): """Swap system memory as a (total, used, free, sin, sout) tuple.""" mem = cext.virtual_mem() - total = mem[2] - free = mem[3] + + total_phys = mem[0] + free_phys = mem[1] + total_system = mem[2] + free_system = mem[3] + + # Despite the name PageFile refers to total system memory here + # thus physical memory values need to be substracted to get swap values + total = total_system - total_phys + free = min(total, free_system - free_phys) used = total - free percent = usage_percent(used, total, round_=1) return _common.sswap(total, used, free, percent, 0, 0) @@ -321,9 +329,9 @@ def cpu_count_logical(): return cext.cpu_count_logical() -def cpu_count_physical(): - """Return the number of physical CPU cores in the system.""" - return cext.cpu_count_phys() +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + return cext.cpu_count_cores() def cpu_stats(): @@ -776,7 +784,7 @@ class Process(object): # 24 = ERROR_TOO_MANY_OPEN_FILES. Not sure why this happens # (perhaps PyPy's JIT delaying garbage collection of files?). if err.errno == 24: - debug("%r forced into AccessDenied" % err) + debug("%r translated into AccessDenied" % err) raise AccessDenied(self.pid, self._name) raise else: @@ -1077,17 +1085,17 @@ class Process(object): @wrap_exceptions def cpu_affinity_get(self): def from_bitmask(x): - return [i for i in xrange(64) if (1 << i) & x] + return [i for i in range(64) if (1 << i) & x] bitmask = cext.proc_cpu_affinity_get(self.pid) return from_bitmask(bitmask) @wrap_exceptions def cpu_affinity_set(self, value): - def to_bitmask(l): - if not l: - raise ValueError("invalid argument %r" % l) + def to_bitmask(ls): + if not ls: + raise ValueError("invalid argument %r" % ls) out = 0 - for b in l: + for b in ls: out |= 2 ** b return out diff --git a/psutil/arch/aix/common.c b/psutil/arch/aix/common.c index 6115a15d..945cbd97 100644 --- a/psutil/arch/aix/common.c +++ b/psutil/arch/aix/common.c @@ -76,4 +76,4 @@ psutil_read_process_table(int * num) { *num = np; return processes; -}
\ No newline at end of file +} diff --git a/psutil/arch/aix/ifaddrs.c b/psutil/arch/aix/ifaddrs.c index 1a819365..1480b60f 100644 --- a/psutil/arch/aix/ifaddrs.c +++ b/psutil/arch/aix/ifaddrs.c @@ -146,4 +146,4 @@ error: close(sd); freeifaddrs(*ifap); return (-1); -}
\ No newline at end of file +} diff --git a/psutil/arch/aix/ifaddrs.h b/psutil/arch/aix/ifaddrs.h index 3920c1cc..e15802bf 100644 --- a/psutil/arch/aix/ifaddrs.h +++ b/psutil/arch/aix/ifaddrs.h @@ -31,5 +31,4 @@ struct ifaddrs { extern int getifaddrs(struct ifaddrs **); extern void freeifaddrs(struct ifaddrs *); - -#endif
\ No newline at end of file +#endif diff --git a/psutil/arch/aix/net_connections.h b/psutil/arch/aix/net_connections.h index 222bcaf3..d57ee428 100644 --- a/psutil/arch/aix/net_connections.h +++ b/psutil/arch/aix/net_connections.h @@ -12,4 +12,4 @@ PyObject* psutil_net_connections(PyObject *self, PyObject *args); -#endif /* __NET_CONNECTIONS_H__ */
\ No newline at end of file +#endif /* __NET_CONNECTIONS_H__ */ diff --git a/psutil/arch/aix/net_kernel_structs.h b/psutil/arch/aix/net_kernel_structs.h index 4e7a088c..7e22a163 100644 --- a/psutil/arch/aix/net_kernel_structs.h +++ b/psutil/arch/aix/net_kernel_structs.h @@ -108,4 +108,4 @@ struct mbuf64 #define m_len m_hdr.mh_len -#endif /* __64BIT__ */
\ No newline at end of file +#endif /* __64BIT__ */ diff --git a/psutil/arch/freebsd/cpu.c b/psutil/arch/freebsd/cpu.c new file mode 100644 index 00000000..f31e9bb0 --- /dev/null +++ b/psutil/arch/freebsd/cpu.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* +System-wide CPU related functions. +Original code was refactored and moved from psutil/arch/freebsd/specific.c +in 2020 (and was moved in there previously already) from cset. +a4c0a0eb0d2a872ab7a45e47fcf37ef1fde5b012 +For reference, here's the git history with original(ish) implementations: +- CPU stats: fb0154ef164d0e5942ac85102ab660b8d2938fbb +- CPU freq: 459556dd1e2979cdee22177339ced0761caf4c83 +- CPU cores: e0d6d7865df84dc9a1d123ae452fd311f79b1dde +*/ + + +#include <Python.h> +#include <sys/sysctl.h> + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + + +PyObject * +psutil_cpu_topology(PyObject *self, PyObject *args) { + void *topology = NULL; + size_t size = 0; + PyObject *py_str; + + if (sysctlbyname("kern.sched.topology_spec", NULL, &size, NULL, 0)) + goto error; + + topology = malloc(size); + if (!topology) { + PyErr_NoMemory(); + return NULL; + } + + if (sysctlbyname("kern.sched.topology_spec", topology, &size, NULL, 0)) + goto error; + + py_str = Py_BuildValue("s", topology); + free(topology); + return py_str; + +error: + if (topology != NULL) + free(topology); + Py_RETURN_NONE; +} + + +PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + unsigned int v_soft; + unsigned int v_intr; + unsigned int v_syscall; + unsigned int v_trap; + unsigned int v_swtch; + size_t size = sizeof(v_soft); + + if (sysctlbyname("vm.stats.sys.v_soft", &v_soft, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_soft')"); + } + if (sysctlbyname("vm.stats.sys.v_intr", &v_intr, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_intr')"); + } + if (sysctlbyname("vm.stats.sys.v_syscall", &v_syscall, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_syscall')"); + } + if (sysctlbyname("vm.stats.sys.v_trap", &v_trap, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_trap')"); + } + if (sysctlbyname("vm.stats.sys.v_swtch", &v_swtch, &size, NULL, 0)) { + return PyErr_SetFromOSErrnoWithSyscall( + "sysctlbyname('vm.stats.sys.v_swtch')"); + } + + return Py_BuildValue( + "IIIII", + v_swtch, // ctx switches + v_intr, // interrupts + v_soft, // software interrupts + v_syscall, // syscalls + v_trap // traps + ); +} + + +/* + * Return frequency information of a given CPU. + * As of Dec 2018 only CPU 0 appears to be supported and all other + * cores match the frequency of CPU 0. + */ +PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + int current; + int core; + char sensor[26]; + char available_freq_levels[1000]; + size_t size = sizeof(current); + + if (! PyArg_ParseTuple(args, "i", &core)) + return NULL; + // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ + sprintf(sensor, "dev.cpu.%d.freq", core); + if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) + goto error; + + size = sizeof(available_freq_levels); + // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ + // In case of failure, an empty string is returned. + sprintf(sensor, "dev.cpu.%d.freq_levels", core); + sysctlbyname(sensor, &available_freq_levels, &size, NULL, 0); + + return Py_BuildValue("is", current, available_freq_levels); + +error: + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "unable to read frequency"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} diff --git a/psutil/arch/freebsd/cpu.h b/psutil/arch/freebsd/cpu.h new file mode 100644 index 00000000..8decd773 --- /dev/null +++ b/psutil/arch/freebsd/cpu.h @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include <Python.h> + +PyObject* psutil_cpu_freq(PyObject* self, PyObject* args); +PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); +PyObject* psutil_cpu_topology(PyObject* self, PyObject* args); diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 2f72be57..f8371e82 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -263,7 +263,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (ret == -1) return NULL; else if (ret == 0) - return NoSuchProcess("psutil_pid_exists"); + return NoSuchProcess("psutil_pid_exists -> 0"); else strcpy(pathname, ""); } @@ -364,37 +364,6 @@ error: } -PyObject * -psutil_cpu_count_phys(PyObject *self, PyObject *args) { - // Return an XML string from which we'll determine the number of - // physical CPU cores in the system. - void *topology = NULL; - size_t size = 0; - PyObject *py_str; - - if (sysctlbyname("kern.sched.topology_spec", NULL, &size, NULL, 0)) - goto error; - - topology = malloc(size); - if (!topology) { - PyErr_NoMemory(); - return NULL; - } - - if (sysctlbyname("kern.sched.topology_spec", topology, &size, NULL, 0)) - goto error; - - py_str = Py_BuildValue("s", topology); - free(topology); - return py_str; - -error: - if (topology != NULL) - free(topology); - Py_RETURN_NONE; -} - - /* * Return virtual memory usage statistics. */ @@ -405,7 +374,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { size_t size = sizeof(total); struct vmtotal vm; int mib[] = {CTL_VM, VM_METER}; - long pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); #if __FreeBSD_version > 702101 long buffers; #else @@ -466,7 +435,7 @@ psutil_swap_mem(PyObject *self, PyObject *args) { struct kvm_swap kvmsw[1]; unsigned int swapin, swapout, nodein, nodeout; size_t size = sizeof(unsigned int); - int pagesize; + long pagesize = psutil_getpagesize(); kd = kvm_open(NULL, _PATH_DEVNULL, NULL, O_RDONLY, "kvm_open failed"); if (kd == NULL) { @@ -500,12 +469,6 @@ psutil_swap_mem(PyObject *self, PyObject *args) { "sysctlbyname('vm.stats.vm.v_vnodeout)'"); } - pagesize = getpagesize(); - if (pagesize <= 0) { - PyErr_SetString(PyExc_ValueError, "invalid getpagesize()"); - return NULL; - } - return Py_BuildValue( "(KKKII)", (unsigned long long)kvmsw[0].ksw_total * pagesize, // total @@ -518,7 +481,7 @@ psutil_swap_mem(PyObject *self, PyObject *args) { } -#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 +#if defined(__FreeBSD_version) && __FreeBSD_version >= 701000 PyObject * psutil_proc_cwd(PyObject *self, PyObject *args) { pid_t pid; @@ -730,7 +693,7 @@ error: PyObject * psutil_proc_memory_maps(PyObject *self, PyObject *args) { // Return a list of tuples for every process memory maps. - //'procstat' cmdline utility has been used as an example. + // 'procstat' cmdline utility has been used as an example. pid_t pid; int ptrwidth; int i, cnt; @@ -796,9 +759,11 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { case KVME_TYPE_DEAD: path = "[dead]"; break; +#ifdef KVME_TYPE_SG case KVME_TYPE_SG: path = "[sg]"; break; +#endif case KVME_TYPE_UNKNOWN: path = "[unknown]"; break; @@ -937,47 +902,6 @@ error: } -PyObject * -psutil_cpu_stats(PyObject *self, PyObject *args) { - unsigned int v_soft; - unsigned int v_intr; - unsigned int v_syscall; - unsigned int v_trap; - unsigned int v_swtch; - size_t size = sizeof(v_soft); - - if (sysctlbyname("vm.stats.sys.v_soft", &v_soft, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('vm.stats.sys.v_soft')"); - } - if (sysctlbyname("vm.stats.sys.v_intr", &v_intr, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('vm.stats.sys.v_intr')"); - } - if (sysctlbyname("vm.stats.sys.v_syscall", &v_syscall, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('vm.stats.sys.v_syscall')"); - } - if (sysctlbyname("vm.stats.sys.v_trap", &v_trap, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('vm.stats.sys.v_trap')"); - } - if (sysctlbyname("vm.stats.sys.v_swtch", &v_swtch, &size, NULL, 0)) { - return PyErr_SetFromOSErrnoWithSyscall( - "sysctlbyname('vm.stats.sys.v_swtch')"); - } - - return Py_BuildValue( - "IIIII", - v_swtch, // ctx switches - v_intr, // interrupts - v_soft, // software interrupts - v_syscall, // syscalls - v_trap // traps - ); -} - - /* * Return battery information. */ @@ -1042,39 +966,87 @@ error: /* - * Return frequency information of a given CPU. - * As of Dec 2018 only CPU 0 appears to be supported and all other - * cores match the frequency of CPU 0. + * An emulation of Linux prlimit(). Returns a (soft, hard) tuple. */ PyObject * -psutil_cpu_freq(PyObject *self, PyObject *args) { - int current; - int core; - char sensor[26]; - char available_freq_levels[1000]; - size_t size = sizeof(current); +psutil_proc_getrlimit(PyObject *self, PyObject *args) { + pid_t pid; + int ret; + int resource; + size_t len; + int name[5]; + struct rlimit rlp; - if (! PyArg_ParseTuple(args, "i", &core)) + if (! PyArg_ParseTuple(args, _Py_PARSE_PID "i", &pid, &resource)) return NULL; - // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ - sprintf(sensor, "dev.cpu.%d.freq", core); - if (sysctlbyname(sensor, ¤t, &size, NULL, 0)) - goto error; - size = sizeof(available_freq_levels); - // https://www.unix.com/man-page/FreeBSD/4/cpufreq/ - // In case of failure, an empty string is returned. - sprintf(sensor, "dev.cpu.%d.freq_levels", core); - sysctlbyname(sensor, &available_freq_levels, &size, NULL, 0); + name[0] = CTL_KERN; + name[1] = KERN_PROC; + name[2] = KERN_PROC_RLIMIT; + name[3] = pid; + name[4] = resource; + len = sizeof(rlp); - return Py_BuildValue("is", current, available_freq_levels); + ret = sysctl(name, 5, &rlp, &len, NULL, 0); + if (ret == -1) + return PyErr_SetFromErrno(PyExc_OSError); -error: - if (errno == ENOENT) - PyErr_SetString(PyExc_NotImplementedError, "unable to read frequency"); - else - PyErr_SetFromErrno(PyExc_OSError); - return NULL; +#if defined(HAVE_LONG_LONG) + return Py_BuildValue("LL", + (PY_LONG_LONG) rlp.rlim_cur, + (PY_LONG_LONG) rlp.rlim_max); +#else + return Py_BuildValue("ll", + (long) rlp.rlim_cur, + (long) rlp.rlim_max); +#endif +} + + +/* + * An emulation of Linux prlimit() (set). + */ +PyObject * +psutil_proc_setrlimit(PyObject *self, PyObject *args) { + pid_t pid; + int ret; + int resource; + int name[5]; + struct rlimit new; + struct rlimit *newp = NULL; + PyObject *py_soft = NULL; + PyObject *py_hard = NULL; + + if (! PyArg_ParseTuple( + args, _Py_PARSE_PID "iOO", &pid, &resource, &py_soft, &py_hard)) + return NULL; + + name[0] = CTL_KERN; + name[1] = KERN_PROC; + name[2] = KERN_PROC_RLIMIT; + name[3] = pid; + name[4] = resource; + +#if defined(HAVE_LONG_LONG) + new.rlim_cur = PyLong_AsLongLong(py_soft); + if (new.rlim_cur == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; + new.rlim_max = PyLong_AsLongLong(py_hard); + if (new.rlim_max == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; +#else + new.rlim_cur = PyLong_AsLong(py_soft); + if (new.rlim_cur == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; + new.rlim_max = PyLong_AsLong(py_hard); + if (new.rlim_max == (rlim_t) - 1 && PyErr_Occurred()) + return NULL; +#endif + newp = &new; + ret = sysctl(name, 5, NULL, 0, newp, sizeof(*newp)); + if (ret == -1) + return PyErr_SetFromErrno(PyExc_OSError); + Py_RETURN_NONE; } diff --git a/psutil/arch/freebsd/specific.h b/psutil/arch/freebsd/specific.h index f05e656e..f5a85e30 100644 --- a/psutil/arch/freebsd/specific.h +++ b/psutil/arch/freebsd/specific.h @@ -11,8 +11,6 @@ typedef struct kinfo_proc kinfo_proc; int psutil_get_proc_list(struct kinfo_proc **procList, size_t *procCount); int psutil_kinfo_proc(const pid_t pid, struct kinfo_proc *proc); -// -PyObject* psutil_cpu_count_phys(PyObject* self, PyObject* args); PyObject* psutil_disk_io_counters(PyObject* self, PyObject* args); PyObject* psutil_get_cmdline(long pid); PyObject* psutil_per_cpu_times(PyObject* self, PyObject* args); @@ -20,16 +18,14 @@ PyObject* psutil_proc_cpu_affinity_get(PyObject* self, PyObject* args); PyObject* psutil_proc_cpu_affinity_set(PyObject* self, PyObject* args); PyObject* psutil_proc_cwd(PyObject* self, PyObject* args); PyObject* psutil_proc_exe(PyObject* self, PyObject* args); +PyObject* psutil_proc_getrlimit(PyObject* self, PyObject* args); PyObject* psutil_proc_memory_maps(PyObject* self, PyObject* args); PyObject* psutil_proc_num_fds(PyObject* self, PyObject* args); PyObject* psutil_proc_num_threads(PyObject* self, PyObject* args); +PyObject* psutil_proc_setrlimit(PyObject* self, PyObject* args); PyObject* psutil_proc_threads(PyObject* self, PyObject* args); +PyObject* psutil_sensors_battery(PyObject* self, PyObject* args); +PyObject* psutil_sensors_cpu_temperature(PyObject* self, PyObject* args); PyObject* psutil_swap_mem(PyObject* self, PyObject* args); PyObject* psutil_virtual_mem(PyObject* self, PyObject* args); -PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); PyObject* psutil_disk_swaps(PyObject* self, PyObject* args); -#if defined(PSUTIL_FREEBSD) -PyObject* psutil_sensors_battery(PyObject* self, PyObject* args); -PyObject* psutil_sensors_cpu_temperature(PyObject* self, PyObject* args); -PyObject* psutil_cpu_freq(PyObject* self, PyObject* args); -#endif diff --git a/psutil/arch/freebsd/sys_socks.c b/psutil/arch/freebsd/sys_socks.c index ab61f393..9f7cf8d5 100644 --- a/psutil/arch/freebsd/sys_socks.c +++ b/psutil/arch/freebsd/sys_socks.c @@ -16,6 +16,9 @@ #include <sys/un.h> #include <sys/unpcb.h> #include <sys/sysctl.h> +#if defined(__FreeBSD_version) && __FreeBSD_version < 800000 +#include <netinet/in_systm.h> +#endif #include <netinet/in.h> // for xinpcb struct #include <netinet/ip.h> #include <netinet/in_pcb.h> @@ -30,7 +33,7 @@ static int psutil_nxfiles; int -psutil_populate_xfiles() { +psutil_populate_xfiles(void) { size_t len; if ((psutil_xfiles = malloc(len = sizeof *psutil_xfiles)) == NULL) { diff --git a/psutil/arch/netbsd/socks.c b/psutil/arch/netbsd/socks.c index f370f094..08b0b7e6 100644 --- a/psutil/arch/netbsd/socks.c +++ b/psutil/arch/netbsd/socks.c @@ -154,7 +154,7 @@ psutil_get_files(void) { // debug struct kif *k; SLIST_FOREACH(k, &kihead, kifs) { - printf("%d\n", k->kif->ki_pid); + printf("%d\n", k->kif->ki_pid); // NOQA } */ @@ -206,17 +206,6 @@ psutil_get_sockets(const char *name) { kpcb->kpcb = &kp[j]; SLIST_INSERT_HEAD(&kpcbhead, kpcb, kpcbs); } - - /* - // debug - struct kif *k; - struct kpcb *k; - SLIST_FOREACH(k, &kpcbhead, kpcbs) { - printf("ki_type: %d\n", k->kpcb->ki_type); - printf("ki_family: %d\n", k->kpcb->ki_family); - } - */ - return 0; } diff --git a/psutil/arch/netbsd/specific.c b/psutil/arch/netbsd/specific.c index 9dab3618..4e286e5e 100644 --- a/psutil/arch/netbsd/specific.c +++ b/psutil/arch/netbsd/specific.c @@ -126,7 +126,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { int name[] = { CTL_KERN, KERN_PROC_ARGS, pid, KERN_PROC_CWD}; if (sysctl(name, 4, path, &pathlen, NULL, 0) != 0) { if (errno == ENOENT) - NoSuchProcess(""); + NoSuchProcess("sysctl -> ENOENT"); else PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -142,7 +142,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { free(buf); if (len == -1) { if (errno == ENOENT) - NoSuchProcess("readlink (ENOENT)"); + NoSuchProcess("readlink -> ENOENT"); else PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -198,7 +198,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (ret == -1) return NULL; else if (ret == 0) - return NoSuchProcess("psutil_pid_exists"); + return NoSuchProcess("psutil_pid_exists -> 0"); else strcpy(pathname, ""); } @@ -445,7 +445,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { size_t size; struct uvmexp_sysctl uv; int mib[] = {CTL_VM, VM_UVMEXP2}; - long pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); size = sizeof(uv); if (sysctl(mib, 2, &uv, &size, NULL, 0) < 0) { @@ -472,6 +472,7 @@ psutil_swap_mem(PyObject *self, PyObject *args) { uint64_t swap_total, swap_free; struct swapent *swdev; int nswap, i; + long pagesize = psutil_getpagesize(); nswap = swapctl(SWAP_NSWAP, 0, 0); if (nswap == 0) { @@ -505,7 +506,6 @@ psutil_swap_mem(PyObject *self, PyObject *args) { size_t size = sizeof(total); struct uvmexp_sysctl uv; int mib[] = {CTL_VM, VM_UVMEXP2}; - long pagesize = getpagesize(); size = sizeof(uv); if (sysctl(mib, 2, &uv, &size, NULL, 0) < 0) { PyErr_SetFromErrno(PyExc_OSError); diff --git a/psutil/arch/openbsd/specific.c b/psutil/arch/openbsd/specific.c index 0b4543a6..080a0fe4 100644 --- a/psutil/arch/openbsd/specific.c +++ b/psutil/arch/openbsd/specific.c @@ -47,20 +47,6 @@ // ============================================================================ -static void -convert_kvm_err(const char *syscall, char *errbuf) { - char fullmsg[8192]; - - sprintf(fullmsg, "(originated from %s: %s)", syscall, errbuf); - if (strstr(errbuf, "Permission denied") != NULL) - AccessDenied(fullmsg); - else if (strstr(errbuf, "Operation not permitted") != NULL) - AccessDenied(fullmsg); - else - PyErr_Format(PyExc_RuntimeError, fullmsg); -} - - int psutil_kinfo_proc(pid_t pid, struct kinfo_proc *proc) { // Fills a kinfo_proc struct based on process pid. @@ -311,7 +297,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { struct uvmexp uvmexp; struct bcachestats bcstats; struct vmtotal vmdata; - long pagesize = getpagesize(); + long pagesize = psutil_getpagesize(); size = sizeof(total_physmem); if (sysctl(physmem_mib, 2, &total_physmem, &size, NULL, 0) < 0) { diff --git a/psutil/arch/osx/cpu.c b/psutil/arch/osx/cpu.c new file mode 100644 index 00000000..37141a2d --- /dev/null +++ b/psutil/arch/osx/cpu.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* +System-wide CPU related functions. + +Original code was refactored and moved from psutil/_psutil_osx.c in 2020 +right before a4c0a0eb0d2a872ab7a45e47fcf37ef1fde5b012. +For reference, here's the git history with original implementations: + +- CPU count logical: 3d291d425b856077e65163e43244050fb188def1 +- CPU count physical: 4263e354bb4984334bc44adf5dd2f32013d69fba +- CPU times: 32488bdf54aed0f8cef90d639c1667ffaa3c31c7 +- CPU stat: fa00dfb961ef63426c7818899340866ced8d2418 +- CPU frequency: 6ba1ac4ebfcd8c95fca324b15606ab0ec1412d39 +*/ + +#include <Python.h> +#include <sys/sysctl.h> +#include <sys/vmmeter.h> + +#include <mach/mach_error.h> +#include <mach/mach_host.h> +#include <mach/mach_port.h> + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + + + +PyObject * +psutil_cpu_count_logical(PyObject *self, PyObject *args) { + int num; + size_t size = sizeof(int); + + if (sysctlbyname("hw.logicalcpu", &num, &size, NULL, 2)) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", num); +} + + +PyObject * +psutil_cpu_count_cores(PyObject *self, PyObject *args) { + int num; + size_t size = sizeof(int); + + if (sysctlbyname("hw.physicalcpu", &num, &size, NULL, 0)) + Py_RETURN_NONE; // mimic os.cpu_count() + else + return Py_BuildValue("i", num); +} + + +PyObject * +psutil_cpu_times(PyObject *self, PyObject *args) { + mach_msg_type_number_t count = HOST_CPU_LOAD_INFO_COUNT; + kern_return_t error; + host_cpu_load_info_data_t r_load; + + mach_port_t host_port = mach_host_self(); + error = host_statistics(host_port, HOST_CPU_LOAD_INFO, + (host_info_t)&r_load, &count); + if (error != KERN_SUCCESS) { + return PyErr_Format( + PyExc_RuntimeError, + "host_statistics(HOST_CPU_LOAD_INFO) syscall failed: %s", + mach_error_string(error)); + } + mach_port_deallocate(mach_task_self(), host_port); + + return Py_BuildValue( + "(dddd)", + (double)r_load.cpu_ticks[CPU_STATE_USER] / CLK_TCK, + (double)r_load.cpu_ticks[CPU_STATE_NICE] / CLK_TCK, + (double)r_load.cpu_ticks[CPU_STATE_SYSTEM] / CLK_TCK, + (double)r_load.cpu_ticks[CPU_STATE_IDLE] / CLK_TCK + ); +} + + +PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + struct vmmeter vmstat; + kern_return_t ret; + mach_msg_type_number_t count = sizeof(vmstat) / sizeof(integer_t); + mach_port_t mport = mach_host_self(); + + ret = host_statistics(mport, HOST_VM_INFO, (host_info_t)&vmstat, &count); + if (ret != KERN_SUCCESS) { + PyErr_Format( + PyExc_RuntimeError, + "host_statistics(HOST_VM_INFO) failed: %s", + mach_error_string(ret)); + return NULL; + } + mach_port_deallocate(mach_task_self(), mport); + + return Py_BuildValue( + "IIIII", + vmstat.v_swtch, // ctx switches + vmstat.v_intr, // interrupts + vmstat.v_soft, // software interrupts + vmstat.v_syscall, // syscalls + vmstat.v_trap // traps + ); +} + + +PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + unsigned int curr; + int64_t min = 0; + int64_t max = 0; + int mib[2]; + size_t len = sizeof(curr); + size_t size = sizeof(min); + + // also availble as "hw.cpufrequency" but it's deprecated + mib[0] = CTL_HW; + mib[1] = HW_CPU_FREQ; + + if (sysctl(mib, 2, &curr, &len, NULL, 0) < 0) + return PyErr_SetFromOSErrnoWithSyscall("sysctl(HW_CPU_FREQ)"); + + if (sysctlbyname("hw.cpufrequency_min", &min, &size, NULL, 0)) + psutil_debug("sysct('hw.cpufrequency_min') failed (set to 0)"); + + if (sysctlbyname("hw.cpufrequency_max", &max, &size, NULL, 0)) + psutil_debug("sysctl('hw.cpufrequency_min') failed (set to 0)"); + + return Py_BuildValue( + "IKK", + curr / 1000 / 1000, + min / 1000 / 1000, + max / 1000 / 1000); +} diff --git a/psutil/arch/osx/cpu.h b/psutil/arch/osx/cpu.h new file mode 100644 index 00000000..aac0f809 --- /dev/null +++ b/psutil/arch/osx/cpu.h @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2009, Jay Loden, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include <Python.h> + +PyObject *psutil_cpu_count_logical(PyObject *self, PyObject *args); +PyObject *psutil_cpu_count_cores(PyObject *self, PyObject *args); +PyObject *psutil_cpu_times(PyObject *self, PyObject *args); +PyObject *psutil_cpu_freq(PyObject *self, PyObject *args); +PyObject *psutil_cpu_stats(PyObject *self, PyObject *args); diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 4b84a723..6ab42750 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -9,13 +9,7 @@ #include <Python.h> -#include <assert.h> #include <errno.h> -#include <limits.h> // for INT_MAX -#include <stdbool.h> -#include <stdlib.h> -#include <stdio.h> -#include <signal.h> #include <sys/sysctl.h> #include <libproc.h> @@ -23,6 +17,7 @@ #include "../../_psutil_posix.h" #include "process_info.h" + /* * Returns a list of all BSD processes on the system. This routine * allocates the list and puts it in *procList and a count of the @@ -33,16 +28,15 @@ */ int psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { - int mib3[3] = { CTL_KERN, KERN_PROC, KERN_PROC_ALL }; + int mib[3]; size_t size, size2; void *ptr; int err; int lim = 8; // some limit - assert( procList != NULL); - assert(*procList == NULL); - assert(procCount != NULL); - + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_ALL; *procCount = 0; /* @@ -59,7 +53,7 @@ psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { */ while (lim-- > 0) { size = 0; - if (sysctl((int *)mib3, 3, NULL, &size, NULL, 0) == -1) { + if (sysctl((int *)mib, 3, NULL, &size, NULL, 0) == -1) { PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROC_ALL)"); return 1; } @@ -79,7 +73,7 @@ psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { return 1; } - if (sysctl((int *)mib3, 3, ptr, &size, NULL, 0) == -1) { + if (sysctl((int *)mib, 3, ptr, &size, NULL, 0) == -1) { err = errno; free(ptr); if (err != ENOMEM) { @@ -104,12 +98,15 @@ psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { // Read the maximum argument size for processes -int -psutil_get_argmax() { +static int +psutil_sysctl_argmax() { int argmax; - int mib[] = { CTL_KERN, KERN_ARGMAX }; + int mib[2]; size_t size = sizeof(argmax); + mib[0] = CTL_KERN; + mib[1] = KERN_ARGMAX; + if (sysctl(mib, 2, &argmax, &size, NULL, 0) == 0) return argmax; PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_ARGMAX)"); @@ -117,6 +114,40 @@ psutil_get_argmax() { } +// Read process argument space. +static int +psutil_sysctl_procargs(pid_t pid, char *procargs, size_t argmax) { + int mib[3]; + + mib[0] = CTL_KERN; + mib[1] = KERN_PROCARGS2; + mib[2] = pid; + + if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { + if (psutil_pid_exists(pid) == 0) { + NoSuchProcess("psutil_pid_exists -> 0"); + return 1; + } + // In case of zombie process we'll get EINVAL. We translate it + // to NSP and _psosx.py will translate it to ZP. + if (errno == EINVAL) { + psutil_debug("sysctl(KERN_PROCARGS2) -> EINVAL translated to NSP"); + NoSuchProcess("sysctl(KERN_PROCARGS2) -> EINVAL"); + return 1; + } + // There's nothing we can do other than raising AD. + if (errno == EIO) { + psutil_debug("sysctl(KERN_PROCARGS2) -> EIO translated to AD"); + AccessDenied("sysctl(KERN_PROCARGS2) -> EIO"); + return 1; + } + PyErr_SetFromOSErrnoWithSyscall("sysctl(KERN_PROCARGS2)"); + return 1; + } + return 0; +} + + // Return 1 if pid refers to a zombie process else 0. int psutil_is_zombie(pid_t pid) { @@ -128,11 +159,9 @@ psutil_is_zombie(pid_t pid) { } - // return process args as a python list PyObject * psutil_get_cmdline(pid_t pid) { - int mib[3]; int nargs; size_t len; char *procargs = NULL; @@ -149,7 +178,7 @@ psutil_get_cmdline(pid_t pid) { return Py_BuildValue("[]"); // read argmax and allocate memory for argument space. - argmax = psutil_get_argmax(); + argmax = psutil_sysctl_argmax(); if (! argmax) goto error; @@ -159,19 +188,8 @@ psutil_get_cmdline(pid_t pid) { goto error; } - // read argument space - mib[0] = CTL_KERN; - mib[1] = KERN_PROCARGS2; - mib[2] = pid; - if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { - // In case of zombie process we'll get EINVAL. We translate it - // to NSP and _psosx.py will translate it to ZP. - if ((errno == EINVAL) && (psutil_pid_exists(pid))) - NoSuchProcess("sysctl"); - else - PyErr_SetFromErrno(PyExc_OSError); + if (psutil_sysctl_procargs(pid, procargs, argmax) != 0) goto error; - } arg_end = &procargs[argmax]; // copy the number of arguments to nargs @@ -226,7 +244,6 @@ error: // return process environment as a python string PyObject * psutil_get_environ(pid_t pid) { - int mib[3]; int nargs; char *procargs = NULL; char *procenv = NULL; @@ -241,7 +258,7 @@ psutil_get_environ(pid_t pid) { goto empty; // read argmax and allocate memory for argument space. - argmax = psutil_get_argmax(); + argmax = psutil_sysctl_argmax(); if (! argmax) goto error; @@ -251,19 +268,8 @@ psutil_get_environ(pid_t pid) { goto error; } - // read argument space - mib[0] = CTL_KERN; - mib[1] = KERN_PROCARGS2; - mib[2] = pid; - if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { - // In case of zombie process we'll get EINVAL. We translate it - // to NSP and _psosx.py will translate it to ZP. - if ((errno == EINVAL) && (psutil_pid_exists(pid))) - NoSuchProcess("sysctl"); - else - PyErr_SetFromErrno(PyExc_OSError); + if (psutil_sysctl_procargs(pid, procargs, argmax) != 0) goto error; - } arg_end = &procargs[argmax]; // copy the number of arguments to nargs @@ -299,12 +305,9 @@ psutil_get_environ(pid_t pid) { while (*arg_ptr != '\0' && arg_ptr < arg_end) { char *s = memchr(arg_ptr + 1, '\0', arg_end - arg_ptr); - if (s == NULL) break; - memcpy(procenv + (arg_ptr - env_start), arg_ptr, s - arg_ptr); - arg_ptr = s + 1; } @@ -320,7 +323,6 @@ psutil_get_environ(pid_t pid) { free(procargs); free(procenv); - return py_ret; empty: @@ -353,13 +355,13 @@ psutil_get_kinfo_proc(pid_t pid, struct kinfo_proc *kp) { // now read the data from sysctl if (sysctl(mib, 4, kp, &len, NULL, 0) == -1) { // raise an exception and throw errno as the error - PyErr_SetFromErrno(PyExc_OSError); + PyErr_SetFromOSErrnoWithSyscall("sysctl"); return -1; } // sysctl succeeds but len is zero, happens when process has gone away if (len == 0) { - NoSuchProcess("sysctl (len == 0)"); + NoSuchProcess("sysctl(kinfo_proc), len == 0"); return -1; } return 0; @@ -368,15 +370,23 @@ psutil_get_kinfo_proc(pid_t pid, struct kinfo_proc *kp) { /* * A wrapper around proc_pidinfo(). - * Returns 0 on failure (and Python exception gets already set). + * https://opensource.apple.com/source/xnu/xnu-2050.7.9/bsd/kern/proc_info.c + * Returns 0 on failure. */ int psutil_proc_pidinfo(pid_t pid, int flavor, uint64_t arg, void *pti, int size) { errno = 0; - int ret = proc_pidinfo(pid, flavor, arg, pti, size); - if ((ret <= 0) || ((unsigned long)ret < sizeof(pti))) { + int ret; + + ret = proc_pidinfo(pid, flavor, arg, pti, size); + if (ret <= 0) { psutil_raise_for_pid(pid, "proc_pidinfo()"); return 0; } + if ((unsigned long)ret < sizeof(pti)) { + psutil_raise_for_pid( + pid, "proc_pidinfo() return size < sizeof(struct_pointer)"); + return 0; + } return ret; } diff --git a/psutil/arch/osx/process_info.h b/psutil/arch/osx/process_info.h index 35755247..ffa6230f 100644 --- a/psutil/arch/osx/process_info.h +++ b/psutil/arch/osx/process_info.h @@ -8,7 +8,6 @@ typedef struct kinfo_proc kinfo_proc; -int psutil_get_argmax(void); int psutil_is_zombie(pid_t pid); int psutil_get_kinfo_proc(pid_t pid, struct kinfo_proc *kp); int psutil_get_proc_list(kinfo_proc **procList, size_t *procCount); diff --git a/psutil/arch/solaris/environ.c b/psutil/arch/solaris/environ.c index eb0fa2ab..482fe1fc 100644 --- a/psutil/arch/solaris/environ.c +++ b/psutil/arch/solaris/environ.c @@ -11,8 +11,8 @@ #include <Python.h> #if !defined(_LP64) && _FILE_OFFSET_BITS == 64 -# undef _FILE_OFFSET_BITS -# undef _LARGEFILE64_SOURCE + #undef _FILE_OFFSET_BITS + #undef _LARGEFILE64_SOURCE #endif #include <sys/types.h> diff --git a/psutil/arch/windows/cpu.c b/psutil/arch/windows/cpu.c index 18f32e59..355de6df 100644 --- a/psutil/arch/windows/cpu.c +++ b/psutil/arch/windows/cpu.c @@ -177,11 +177,10 @@ psutil_cpu_count_logical(PyObject *self, PyObject *args) { /* - * Return the number of physical CPU cores (hyper-thread CPUs count - * is excluded). + * Return the number of CPU cores (non hyper-threading). */ PyObject * -psutil_cpu_count_phys(PyObject *self, PyObject *args) { +psutil_cpu_count_cores(PyObject *self, PyObject *args) { DWORD rc; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX buffer = NULL; PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ptr = NULL; @@ -196,7 +195,7 @@ psutil_cpu_count_phys(PyObject *self, PyObject *args) { // than 64 CPUs. See: // https://bugs.python.org/issue33166 if (GetLogicalProcessorInformationEx == NULL) { - psutil_debug("Win < 7; cpu_count_phys() forced to None"); + psutil_debug("Win < 7; cpu_count_cores() forced to None"); Py_RETURN_NONE; } diff --git a/psutil/arch/windows/cpu.h b/psutil/arch/windows/cpu.h index d88c2212..1ef3ff1f 100644 --- a/psutil/arch/windows/cpu.h +++ b/psutil/arch/windows/cpu.h @@ -7,7 +7,7 @@ #include <Python.h> PyObject *psutil_cpu_count_logical(PyObject *self, PyObject *args); -PyObject *psutil_cpu_count_phys(PyObject *self, PyObject *args); +PyObject *psutil_cpu_count_cores(PyObject *self, PyObject *args); PyObject *psutil_cpu_freq(PyObject *self, PyObject *args); PyObject *psutil_cpu_stats(PyObject *self, PyObject *args); PyObject *psutil_cpu_times(PyObject *self, PyObject *args); diff --git a/psutil/arch/windows/disk.c b/psutil/arch/windows/disk.c index 0cf95331..c3a38082 100644 --- a/psutil/arch/windows/disk.c +++ b/psutil/arch/windows/disk.c @@ -201,11 +201,12 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { int type; int ret; unsigned int old_mode = 0; - char opts[20]; + char opts[50]; HANDLE mp_h; BOOL mp_flag= TRUE; LPTSTR fs_type[MAX_PATH + 1] = { 0 }; DWORD pflags = 0; + DWORD lpMaximumComponentLength = 0; // max file name PyObject *py_all; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; @@ -257,8 +258,14 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { } ret = GetVolumeInformation( - (LPCTSTR)drive_letter, NULL, _ARRAYSIZE(drive_letter), - NULL, NULL, &pflags, (LPTSTR)fs_type, _ARRAYSIZE(fs_type)); + (LPCTSTR)drive_letter, + NULL, + _ARRAYSIZE(drive_letter), + NULL, + &lpMaximumComponentLength, + &pflags, + (LPTSTR)fs_type, + _ARRAYSIZE(fs_type)); if (ret == 0) { // We might get here in case of a floppy hard drive, in // which case the error is (21, "device not ready"). @@ -274,6 +281,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strcat_s(opts, _countof(opts), "rw"); if (pflags & FILE_VOLUME_IS_COMPRESSED) strcat_s(opts, _countof(opts), ",compressed"); + if (pflags & FILE_READ_ONLY_VOLUME) + strcat_s(opts, _countof(opts), ",readonly"); // Check for mount points on this volume and add/get info // (checks first to know if we can even have mount points) @@ -281,18 +290,21 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { mp_h = FindFirstVolumeMountPoint( drive_letter, mp_buf, MAX_PATH); if (mp_h != INVALID_HANDLE_VALUE) { + mp_flag = TRUE; while (mp_flag) { - // Append full mount path with drive letter strcpy_s(mp_path, _countof(mp_path), drive_letter); strcat_s(mp_path, _countof(mp_path), mp_buf); py_tuple = Py_BuildValue( - "(ssss)", + "(ssssIi)", drive_letter, mp_path, - fs_type, // Typically NTFS - opts); + fs_type, // typically "NTFS" + opts, + lpMaximumComponentLength, // max file length + MAX_PATH // max path length + ); if (!py_tuple || PyList_Append(py_retlist, py_tuple) == -1) { @@ -317,11 +329,14 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strcat_s(opts, _countof(opts), psutil_get_drive_type(type)); py_tuple = Py_BuildValue( - "(ssss)", + "(ssssIi)", drive_letter, drive_letter, fs_type, // either FAT, FAT32, NTFS, HPFS, CDFS, UDF or NWFS - opts); + opts, + lpMaximumComponentLength, // max file length + MAX_PATH // max path length + ); if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) @@ -433,7 +448,7 @@ error: If no match is found return an empty string. */ PyObject * -psutil_win32_QueryDosDevice(PyObject *self, PyObject *args) { +psutil_QueryDosDevice(PyObject *self, PyObject *args) { LPCTSTR lpDevicePath; TCHAR d = TEXT('A'); TCHAR szBuff[5]; diff --git a/psutil/arch/windows/disk.h b/psutil/arch/windows/disk.h index aa22ad95..51852d0d 100644 --- a/psutil/arch/windows/disk.h +++ b/psutil/arch/windows/disk.h @@ -9,5 +9,6 @@ PyObject *psutil_disk_io_counters(PyObject *self, PyObject *args); PyObject *psutil_disk_partitions(PyObject *self, PyObject *args); PyObject *psutil_disk_usage(PyObject *self, PyObject *args); +PyObject *psutil_QueryDosDevice(PyObject *self, PyObject *args); PyObject *psutil_disk_swaps(PyObject *self, PyObject *args); -PyObject *psutil_win32_QueryDosDevice(PyObject *self, PyObject *args); + diff --git a/psutil/arch/windows/net.c b/psutil/arch/windows/net.c index f0572d52..8d8f7d1c 100644 --- a/psutil/arch/windows/net.c +++ b/psutil/arch/windows/net.c @@ -16,40 +16,35 @@ static PIP_ADAPTER_ADDRESSES -psutil_get_nic_addresses() { - // allocate a 15 KB buffer to start with - int outBufLen = 15000; - DWORD dwRetVal = 0; - ULONG attempts = 0; - PIP_ADAPTER_ADDRESSES pAddresses = NULL; - - do { - pAddresses = (IP_ADAPTER_ADDRESSES *) malloc(outBufLen); - if (pAddresses == NULL) { - PyErr_NoMemory(); - return NULL; - } - - dwRetVal = GetAdaptersAddresses(AF_UNSPEC, 0, NULL, pAddresses, - &outBufLen); - if (dwRetVal == ERROR_BUFFER_OVERFLOW) { - free(pAddresses); - pAddresses = NULL; - } - else { - break; - } - - attempts++; - } while ((dwRetVal == ERROR_BUFFER_OVERFLOW) && (attempts < 3)); +psutil_get_nic_addresses(void) { + ULONG bufferLength = 0; + PIP_ADAPTER_ADDRESSES buffer; + + if (GetAdaptersAddresses(AF_UNSPEC, 0, NULL, NULL, &bufferLength) + != ERROR_BUFFER_OVERFLOW) + { + PyErr_SetString(PyExc_RuntimeError, + "GetAdaptersAddresses() syscall failed."); + return NULL; + } - if (dwRetVal != NO_ERROR) { - PyErr_SetString( - PyExc_RuntimeError, "GetAdaptersAddresses() syscall failed."); + buffer = malloc(bufferLength); + if (buffer == NULL) { + PyErr_NoMemory(); + return NULL; + } + memset(buffer, 0, bufferLength); + + if (GetAdaptersAddresses(AF_UNSPEC, 0, NULL, buffer, &bufferLength) + != ERROR_SUCCESS) + { + free(buffer); + PyErr_SetString(PyExc_RuntimeError, + "GetAdaptersAddresses() syscall failed."); return NULL; } - return pAddresses; + return buffer; } @@ -189,11 +184,11 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { for (i = 0; i < (int) pCurrAddresses->PhysicalAddressLength; i++) { if (i == (pCurrAddresses->PhysicalAddressLength - 1)) { sprintf_s(ptr, _countof(buff_macaddr), "%.2X\n", - (int)pCurrAddresses->PhysicalAddress[i]); + (int)pCurrAddresses->PhysicalAddress[i]); } else { sprintf_s(ptr, _countof(buff_macaddr), "%.2X-", - (int)pCurrAddresses->PhysicalAddress[i]); + (int)pCurrAddresses->PhysicalAddress[i]); } ptr += 3; } @@ -323,8 +318,7 @@ error: /* * Provides stats about NIC interfaces installed on the system. - * TODO: get 'duplex' (currently it's hard coded to '2', aka - 'full duplex') + * TODO: get 'duplex' (currently it's hard coded to '2', aka 'full duplex') */ PyObject * psutil_net_if_stats(PyObject *self, PyObject *args) { @@ -401,7 +395,7 @@ psutil_net_if_stats(PyObject *self, PyObject *args) { } // is up? - if((pIfRow->dwOperStatus == MIB_IF_OPER_STATUS_CONNECTED || + if ((pIfRow->dwOperStatus == MIB_IF_OPER_STATUS_CONNECTED || pIfRow->dwOperStatus == MIB_IF_OPER_STATUS_OPERATIONAL) && pIfRow->dwAdminStatus == 1 ) { py_is_up = Py_True; diff --git a/psutil/arch/windows/ntextapi.h b/psutil/arch/windows/ntextapi.h index d585c678..3b271334 100644 --- a/psutil/arch/windows/ntextapi.h +++ b/psutil/arch/windows/ntextapi.h @@ -19,6 +19,12 @@ typedef LONG NTSTATUS; #define STATUS_NOT_FOUND ((NTSTATUS)0xC0000225L) #define STATUS_BUFFER_OVERFLOW ((NTSTATUS)0x80000005L) +// WtsApi32.h +#define WTS_CURRENT_SERVER_HANDLE ((HANDLE)NULL) +#define WINSTATIONNAME_LENGTH 32 +#define DOMAIN_LENGTH 17 +#define USERNAME_LENGTH 20 + // ================================================================ // Enums // ================================================================ @@ -95,6 +101,53 @@ typedef enum _KWAIT_REASON { MaximumWaitReason } KWAIT_REASON, *PKWAIT_REASON; +// users() +typedef enum _WTS_INFO_CLASS { + WTSInitialProgram, + WTSApplicationName, + WTSWorkingDirectory, + WTSOEMId, + WTSSessionId, + WTSUserName, + WTSWinStationName, + WTSDomainName, + WTSConnectState, + WTSClientBuildNumber, + WTSClientName, + WTSClientDirectory, + WTSClientProductId, + WTSClientHardwareId, + WTSClientAddress, + WTSClientDisplay, + WTSClientProtocolType, + WTSIdleTime, + WTSLogonTime, + WTSIncomingBytes, + WTSOutgoingBytes, + WTSIncomingFrames, + WTSOutgoingFrames, + WTSClientInfo, + WTSSessionInfo, + WTSSessionInfoEx, + WTSConfigInfo, + WTSValidationInfo, // Info Class value used to fetch Validation Information through the WTSQuerySessionInformation + WTSSessionAddressV4, + WTSIsRemoteSession +} WTS_INFO_CLASS; + +typedef enum _WTS_CONNECTSTATE_CLASS { + WTSActive, // User logged on to WinStation + WTSConnected, // WinStation connected to client + WTSConnectQuery, // In the process of connecting to client + WTSShadow, // Shadowing another WinStation + WTSDisconnected, // WinStation logged on without client + WTSIdle, // Waiting for client to connect + WTSListen, // WinStation is listening for connection + WTSReset, // WinStation is being reset + WTSDown, // WinStation is down due to error + WTSInit, // WinStation in initialization +} WTS_CONNECTSTATE_CLASS; + // ================================================================ // Structs. // ================================================================ @@ -311,19 +364,44 @@ typedef struct { } RTL_USER_PROCESS_PARAMETERS_, *PRTL_USER_PROCESS_PARAMETERS_; // users() -typedef struct _WINSTATION_INFO { - BYTE Reserved1[72]; - ULONG SessionId; - BYTE Reserved2[4]; - FILETIME ConnectTime; - FILETIME DisconnectTime; - FILETIME LastInputTime; - FILETIME LoginTime; - BYTE Reserved3[1096]; - FILETIME CurrentTime; -} WINSTATION_INFO, *PWINSTATION_INFO; - -// cpu_count_phys() +typedef struct _WTS_SESSION_INFOW { + DWORD SessionId; // session id + LPWSTR pWinStationName; // name of WinStation this session is + // connected to + WTS_CONNECTSTATE_CLASS State; // connection state (see enum) +} WTS_SESSION_INFOW, * PWTS_SESSION_INFOW; + +#define PWTS_SESSION_INFO PWTS_SESSION_INFOW + +typedef struct _WTS_CLIENT_ADDRESS { + DWORD AddressFamily; // AF_INET, AF_INET6, AF_IPX, AF_NETBIOS, AF_UNSPEC + BYTE Address[20]; // client network address +} WTS_CLIENT_ADDRESS, * PWTS_CLIENT_ADDRESS; + +typedef struct _WTSINFOW { + WTS_CONNECTSTATE_CLASS State; // connection state (see enum) + DWORD SessionId; // session id + DWORD IncomingBytes; + DWORD OutgoingBytes; + DWORD IncomingFrames; + DWORD OutgoingFrames; + DWORD IncomingCompressedBytes; + DWORD OutgoingCompressedBytes; + WCHAR WinStationName[WINSTATIONNAME_LENGTH]; + WCHAR Domain[DOMAIN_LENGTH]; + WCHAR UserName[USERNAME_LENGTH + 1];// name of WinStation this session is + // connected to + LARGE_INTEGER ConnectTime; + LARGE_INTEGER DisconnectTime; + LARGE_INTEGER LastInputTime; + LARGE_INTEGER LogonTime; + LARGE_INTEGER CurrentTime; + +} WTSINFOW, * PWTSINFOW; + +#define PWTSINFO PWTSINFOW + +// cpu_count_cores() #if (_WIN32_WINNT < 0x0601) // Windows < 7 (Vista and XP) typedef struct _SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX { LOGICAL_PROCESSOR_RELATIONSHIP Relationship; @@ -561,6 +639,32 @@ DWORD (CALLBACK *_GetActiveProcessorCount) ( #define GetActiveProcessorCount _GetActiveProcessorCount +BOOL(CALLBACK *_WTSQuerySessionInformationW) ( + HANDLE hServer, + DWORD SessionId, + WTS_INFO_CLASS WTSInfoClass, + LPWSTR* ppBuffer, + DWORD* pBytesReturned + ); + +#define WTSQuerySessionInformationW _WTSQuerySessionInformationW + +BOOL(CALLBACK *_WTSEnumerateSessionsW)( + HANDLE hServer, + DWORD Reserved, + DWORD Version, + PWTS_SESSION_INFO* ppSessionInfo, + DWORD* pCount + ); + +#define WTSEnumerateSessionsW _WTSEnumerateSessionsW + +VOID(CALLBACK *_WTSFreeMemory)( + IN PVOID pMemory + ); + +#define WTSFreeMemory _WTSFreeMemory + ULONGLONG (CALLBACK *_GetTickCount64) ( void); diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index f63d4af3..72e3f4d4 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -247,7 +247,7 @@ psutil_get_open_files(DWORD dwPid, HANDLE hProcess) { if (psutil_threaded_get_filename(hFile) != 0) goto error; - if (globalFileName->Length > 0) { + if ((globalFileName != NULL) && (globalFileName->Length > 0)) { py_path = PyUnicode_FromWideChar(globalFileName->Buffer, wcslen(globalFileName->Buffer)); if (! py_path) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 73a69912..d44c4eb7 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -50,6 +50,48 @@ enum psutil_process_data_kind { }; +static void +psutil_convert_winerr(ULONG err, char* syscall) { + char fullmsg[8192]; + + if (err == ERROR_NOACCESS) { + sprintf(fullmsg, "%s -> ERROR_NOACCESS", syscall); + psutil_debug(fullmsg); + AccessDenied(fullmsg); + } + else { + PyErr_SetFromOSErrnoWithSyscall(syscall); + } +} + + +static void +psutil_convert_ntstatus_err(NTSTATUS status, char* syscall) { + ULONG err; + + if (NT_NTWIN32(status)) + err = WIN32_FROM_NTSTATUS(status); + else + err = RtlNtStatusToDosErrorNoTeb(status); + psutil_convert_winerr(err, syscall); +} + + +static void +psutil_giveup_with_ad(NTSTATUS status, char* syscall) { + ULONG err; + char fullmsg[8192]; + + if (NT_NTWIN32(status)) + err = WIN32_FROM_NTSTATUS(status); + else + err = RtlNtStatusToDosErrorNoTeb(status); + sprintf(fullmsg, "%s -> %lu (%s)", syscall, err, strerror(err)); + psutil_debug(fullmsg); + AccessDenied(fullmsg); +} + + /* * Get data from the process with the given pid. The data is returned * in the pdata output member as a nul terminated string which must be @@ -87,16 +129,14 @@ psutil_get_process_data(DWORD pid, http://www.drdobbs.com/embracing-64-bit-windows/184401966 */ SIZE_T size = 0; -#ifndef _WIN64 - static __NtQueryInformationProcess NtWow64QueryInformationProcess64 = NULL; - static _NtWow64ReadVirtualMemory64 NtWow64ReadVirtualMemory64 = NULL; -#endif HANDLE hProcess = NULL; LPCVOID src; WCHAR *buffer = NULL; #ifdef _WIN64 LPVOID ppeb32 = NULL; #else + static __NtQueryInformationProcess NtWow64QueryInformationProcess64 = NULL; + static _NtWow64ReadVirtualMemory64 NtWow64ReadVirtualMemory64 = NULL; PVOID64 src64; BOOL weAreWow64; BOOL theyAreWow64; @@ -133,7 +173,7 @@ psutil_get_process_data(DWORD pid, if (!ReadProcessMemory(hProcess, ppeb32, &peb32, sizeof(peb32), NULL)) { // May fail with ERROR_PARTIAL_COPY, see: // https://github.com/giampaolo/psutil/issues/875 - PyErr_SetFromWindowsErr(0); + psutil_convert_winerr(GetLastError(), "ReadProcessMemory"); goto error; } @@ -146,7 +186,7 @@ psutil_get_process_data(DWORD pid, { // May fail with ERROR_PARTIAL_COPY, see: // https://github.com/giampaolo/psutil/issues/875 - PyErr_SetFromWindowsErr(0); + psutil_convert_winerr(GetLastError(), "ReadProcessMemory"); goto error; } @@ -164,8 +204,19 @@ psutil_get_process_data(DWORD pid, break; } } else -#else - /* 32 bit case. Check if the target is also 32 bit. */ +#else // #ifdef _WIN64 + // 32 bit process. In here we may run into a lot of errors, e.g.: + // * [Error 0] The operation completed successfully + // (originated from NtWow64ReadVirtualMemory64) + // * [Error 998] Invalid access to memory location: + // (originated from NtWow64ReadVirtualMemory64) + // Refs: + // * https://github.com/giampaolo/psutil/issues/1839 + // * https://github.com/giampaolo/psutil/pull/1866 + // Since the following code is quite hackish and fails unpredictably, + // in case of any error from NtWow64* APIs we raise AccessDenied. + + // 32 bit case. Check if the target is also 32 bit. if (!IsWow64Process(GetCurrentProcess(), &weAreWow64) || !IsWow64Process(hProcess, &theyAreWow64)) { PyErr_SetFromOSErrnoWithSyscall("IsWow64Process"); @@ -206,10 +257,14 @@ psutil_get_process_data(DWORD pid, sizeof(pbi64), NULL); if (!NT_SUCCESS(status)) { - psutil_SetFromNTStatusErr( - status, - "NtWow64QueryInformationProcess64(ProcessBasicInformation)" - ); + /* + psutil_convert_ntstatus_err( + status, + "NtWow64QueryInformationProcess64(ProcessBasicInformation)"); + */ + psutil_giveup_with_ad( + status, + "NtWow64QueryInformationProcess64(ProcessBasicInformation)"); goto error; } @@ -221,7 +276,13 @@ psutil_get_process_data(DWORD pid, sizeof(peb64), NULL); if (!NT_SUCCESS(status)) { - psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64"); + /* + psutil_convert_ntstatus_err( + status, "NtWow64ReadVirtualMemory64(pbi64.PebBaseAddress)"); + */ + psutil_giveup_with_ad( + status, + "NtWow64ReadVirtualMemory64(pbi64.PebBaseAddress)"); goto error; } @@ -233,10 +294,13 @@ psutil_get_process_data(DWORD pid, sizeof(procParameters64), NULL); if (!NT_SUCCESS(status)) { - psutil_SetFromNTStatusErr( - status, - "NtWow64ReadVirtualMemory64(ProcessParameters)" - ); + /* + psutil_convert_ntstatus_err( + status, "NtWow64ReadVirtualMemory64(peb64.ProcessParameters)"); + */ + psutil_giveup_with_ad( + status, + "NtWow64ReadVirtualMemory64(peb64.ProcessParameters)"); goto error; } @@ -284,7 +348,7 @@ psutil_get_process_data(DWORD pid, { // May fail with ERROR_PARTIAL_COPY, see: // https://github.com/giampaolo/psutil/issues/875 - PyErr_SetFromWindowsErr(0); + psutil_convert_winerr(GetLastError(), "ReadProcessMemory"); goto error; } @@ -297,7 +361,7 @@ psutil_get_process_data(DWORD pid, { // May fail with ERROR_PARTIAL_COPY, see: // https://github.com/giampaolo/psutil/issues/875 - PyErr_SetFromWindowsErr(0); + psutil_convert_winerr(GetLastError(), "ReadProcessMemory"); goto error; } @@ -343,7 +407,8 @@ psutil_get_process_data(DWORD pid, size, NULL); if (!NT_SUCCESS(status)) { - psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64"); + // psutil_convert_ntstatus_err(status, "NtWow64ReadVirtualMemory64"); + psutil_giveup_with_ad(status, "NtWow64ReadVirtualMemory64"); goto error; } } else @@ -351,7 +416,7 @@ psutil_get_process_data(DWORD pid, if (!ReadProcessMemory(hProcess, src, buffer, size, NULL)) { // May fail with ERROR_PARTIAL_COPY, see: // https://github.com/giampaolo/psutil/issues/875 - PyErr_SetFromWindowsErr(0); + psutil_convert_winerr(GetLastError(), "ReadProcessMemory"); goto error; } @@ -408,7 +473,7 @@ psutil_cmdline_query_proc(DWORD pid, WCHAR **pdata, SIZE_T *psize) { // https://github.com/giampaolo/psutil/issues/1501 if (status == STATUS_NOT_FOUND) { AccessDenied("NtQueryInformationProcess(ProcessBasicInformation) -> " - "STATUS_NOT_FOUND translated into PermissionError"); + "STATUS_NOT_FOUND"); goto error; } @@ -698,16 +763,16 @@ psutil_proc_info(PyObject *self, PyObject *args) { py_retlist = Py_BuildValue( #if defined(_WIN64) - "kkdddiKKKKKK" "kKKKKKKKKK", + "kkdddkKKKKKK" "kKKKKKKKKK", #else - "kkdddiKKKKKK" "kIIIIIIIII", + "kkdddkKKKKKK" "kIIIIIIIII", #endif process->HandleCount, // num handles ctx_switches, // num ctx switches user_time, // cpu user time kernel_time, // cpu kernel time create_time, // create time - (int)process->NumberOfThreads, // num threads + process->NumberOfThreads, // num threads // IO counters process->ReadOperationCount.QuadPart, // io rcount process->WriteOperationCount.QuadPart, // io wcount diff --git a/psutil/arch/windows/process_utils.c b/psutil/arch/windows/process_utils.c index f9d2f2f9..acbda301 100644 --- a/psutil/arch/windows/process_utils.c +++ b/psutil/arch/windows/process_utils.c @@ -61,8 +61,10 @@ psutil_pid_in_pids(DWORD pid) { DWORD i; proclist = psutil_get_pids(&numberOfReturnedPIDs); - if (proclist == NULL) + if (proclist == NULL) { + psutil_debug("psutil_get_pids() failed"); return -1; + } for (i = 0; i < numberOfReturnedPIDs; i++) { if (proclist[i] == pid) { free(proclist); @@ -78,20 +80,36 @@ psutil_pid_in_pids(DWORD pid) { // does return the handle, else return NULL with Python exception set. // This is needed because OpenProcess API sucks. HANDLE -psutil_check_phandle(HANDLE hProcess, DWORD pid) { +psutil_check_phandle(HANDLE hProcess, DWORD pid, int check_exit_code) { DWORD exitCode; if (hProcess == NULL) { if (GetLastError() == ERROR_INVALID_PARAMETER) { // Yeah, this is the actual error code in case of // "no such process". - NoSuchProcess("OpenProcess"); + NoSuchProcess("OpenProcess -> ERROR_INVALID_PARAMETER"); + return NULL; + } + if (GetLastError() == ERROR_SUCCESS) { + // Yeah, it's this bad. + // https://github.com/giampaolo/psutil/issues/1877 + if (psutil_pid_in_pids(pid) == 1) { + psutil_debug("OpenProcess -> ERROR_SUCCESS turned into AD"); + AccessDenied("OpenProcess -> ERROR_SUCCESS"); + } + else { + psutil_debug("OpenProcess -> ERROR_SUCCESS turned into NSP"); + NoSuchProcess("OpenProcess -> ERROR_SUCCESS"); + } return NULL; } PyErr_SetFromOSErrnoWithSyscall("OpenProcess"); return NULL; } + if (check_exit_code == 0) + return hProcess; + if (GetExitCodeProcess(hProcess, &exitCode)) { // XXX - maybe STILL_ACTIVE is not fully reliable as per: // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 @@ -137,7 +155,7 @@ psutil_handle_from_pid(DWORD pid, DWORD access) { return NULL; } - hProcess = psutil_check_phandle(hProcess, pid); + hProcess = psutil_check_phandle(hProcess, pid, 1); return hProcess; } @@ -159,7 +177,7 @@ psutil_pid_is_running(DWORD pid) { if ((hProcess == NULL) && (GetLastError() == ERROR_ACCESS_DENIED)) return 1; - hProcess = psutil_check_phandle(hProcess, pid); + hProcess = psutil_check_phandle(hProcess, pid, 1); if (hProcess != NULL) { CloseHandle(hProcess); return 1; diff --git a/psutil/arch/windows/process_utils.h b/psutil/arch/windows/process_utils.h index a7171c5c..dca7c991 100644 --- a/psutil/arch/windows/process_utils.h +++ b/psutil/arch/windows/process_utils.h @@ -6,6 +6,7 @@ DWORD* psutil_get_pids(DWORD *numberOfReturnedPIDs); HANDLE psutil_handle_from_pid(DWORD pid, DWORD dwDesiredAccess); +HANDLE psutil_check_phandle(HANDLE hProcess, DWORD pid, int check_exit_code); int psutil_pid_is_running(DWORD pid); int psutil_assert_pid_exists(DWORD pid, char *err); int psutil_assert_pid_not_exists(DWORD pid, char *err); diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index a91cb8f7..fa3e646e 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -433,6 +433,7 @@ psutil_winservice_start(PyObject *self, PyObject *args) { goto error; } + CloseServiceHandle(hService); Py_RETURN_NONE; error: diff --git a/psutil/arch/windows/wmi.c b/psutil/arch/windows/wmi.c index 42a70df7..5fad4053 100644 --- a/psutil/arch/windows/wmi.c +++ b/psutil/arch/windows/wmi.c @@ -10,6 +10,8 @@ #include <windows.h> #include <pdh.h> +#include "../../_psutil_common.h" + // We use an exponentially weighted moving average, just like Unix systems do // https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation @@ -30,7 +32,7 @@ double load_avg_5m = 0; double load_avg_15m = 0; -VOID CALLBACK LoadAvgCallback(PVOID hCounter) { +VOID CALLBACK LoadAvgCallback(PVOID hCounter, BOOLEAN timedOut) { PDH_FMT_COUNTERVALUE displayValue; double currentLoad; PDH_STATUS err; @@ -62,22 +64,28 @@ psutil_init_loadavg_counter(PyObject *self, PyObject *args) { HANDLE event; HANDLE waitHandle; - if ((PdhOpenQueryW(NULL, 0, &hQuery)) != ERROR_SUCCESS) - goto error; + if ((PdhOpenQueryW(NULL, 0, &hQuery)) != ERROR_SUCCESS) { + PyErr_Format(PyExc_RuntimeError, "PdhOpenQueryW failed"); + return NULL; + } s = PdhAddEnglishCounterW(hQuery, szCounterPath, 0, &hCounter); - if (s != ERROR_SUCCESS) - goto error; + if (s != ERROR_SUCCESS) { + PyErr_Format(PyExc_RuntimeError, "PdhAddEnglishCounterW failed"); + return NULL; + } event = CreateEventW(NULL, FALSE, FALSE, L"LoadUpdateEvent"); if (event == NULL) { - PyErr_SetFromWindowsErr(GetLastError()); + PyErr_SetFromOSErrnoWithSyscall("CreateEventW"); return NULL; } s = PdhCollectQueryDataEx(hQuery, SAMPLING_INTERVAL, event); - if (s != ERROR_SUCCESS) - goto error; + if (s != ERROR_SUCCESS) { + PyErr_Format(PyExc_RuntimeError, "PdhCollectQueryDataEx failed"); + return NULL; + } ret = RegisterWaitForSingleObject( &waitHandle, @@ -89,15 +97,11 @@ psutil_init_loadavg_counter(PyObject *self, PyObject *args) { WT_EXECUTEDEFAULT); if (ret == 0) { - PyErr_SetFromWindowsErr(GetLastError()); + PyErr_SetFromOSErrnoWithSyscall("RegisterWaitForSingleObject"); return NULL; } Py_RETURN_NONE; - -error: - PyErr_SetFromWindowsErr(0); - return NULL; } diff --git a/psutil/tests/README.rst b/psutil/tests/README.rst index b647d513..9dca1186 100644 --- a/psutil/tests/README.rst +++ b/psutil/tests/README.rst @@ -4,20 +4,11 @@ Instructions for running tests * There are two ways of running tests. As a "user", if psutil is already installed and you just want to test it works:: - python -m psutil.tests --install-deps # install test deps python -m psutil.tests As a "developer", if you have a copy of the source code and you wish to hack on psutil:: - make setup-dev-env # install test deps (+ other things) - make test - -* To run tests on all supported Python versions install tox - (``pip install tox``) then run ``tox`` from within psutil root directory. - -* Every time a commit is pushed tests are automatically run on Travis - (Linux, MACOS) and appveyor (Windows): - - * Travis builds: https://travis-ci.org/giampaolo/psutil - * AppVeyor builds: https://ci.appveyor.com/project/giampaolo/psutil + make setup-dev-env # install missing third-party deps + make test # serial run + make test-parallel # parallel run diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 78258f41..47d3a554 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -9,17 +9,19 @@ Test utilities. """ from __future__ import print_function - import atexit import contextlib import ctypes import errno import functools +import gc +import inspect import os import random import re import select import shutil +import signal import socket import stat import subprocess @@ -28,7 +30,6 @@ import tempfile import textwrap import threading import time -import traceback import warnings from socket import AF_INET from socket import AF_INET6 @@ -36,23 +37,28 @@ from socket import SOCK_STREAM import psutil from psutil import AIX +from psutil import FREEBSD +from psutil import LINUX from psutil import MACOS from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS +from psutil._common import bytes2human +from psutil._common import print_color from psutil._common import supports_ipv6 -from psutil._compat import ChildProcessError from psutil._compat import FileExistsError from psutil._compat import FileNotFoundError from psutil._compat import PY3 +from psutil._compat import range +from psutil._compat import super from psutil._compat import u from psutil._compat import unicode from psutil._compat import which -if sys.version_info < (2, 7): - import unittest2 as unittest # requires "pip install unittest2" -else: +if PY3: import unittest +else: + import unittest2 as unittest # requires "pip install unittest2" try: from unittest import mock # py3 @@ -66,38 +72,42 @@ if sys.version_info >= (3, 4): else: enum = None +if POSIX: + from psutil._psposix import wait_pid + __all__ = [ # constants - 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'MEMORY_TOLERANCE', 'NO_RETRIES', - 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFILE_PREFIX', - 'TESTFN', 'TESTFN_UNICODE', 'TOX', 'TRAVIS', 'CIRRUS', 'CI_TESTING', - 'VALID_PROC_STATUSES', + 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', + 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX', + 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', + 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT', "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", # subprocesses - 'pyrun', 'reap_children', 'get_test_subprocess', 'create_zombie_proc', - 'create_proc_children_pair', + 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', + 'spawn_children_pair', # threads 'ThreadTask' # test utils 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', - 'retry_on_failure', + 'retry_on_failure', 'TestMemoryLeak', 'PsutilTestCase', + 'process_namespace', 'system_namespace', 'print_sysinfo', # install utils 'install_pip', 'install_test_deps', # fs utils 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path', - 'unique_filename', + 'get_testfn', # os - 'get_winver', 'get_kernel_version', + 'get_winver', 'kernel_version', # sync primitives 'call_until', 'wait_for_pid', 'wait_for_file', # network 'check_net_address', - 'get_free_port', 'unix_socket_path', 'bind_socket', 'bind_unix_socket', - 'tcp_socketpair', 'unix_socketpair', 'create_sockets', + 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', + 'unix_socketpair', 'create_sockets', # compat 'reload_module', 'import_module_by_path', # others @@ -111,43 +121,45 @@ __all__ = [ # --- platforms -TOX = os.getenv('TOX') or '' in ('1', 'true') PYPY = '__pypy__' in sys.builtin_module_names # whether we're running this test suite on a Continuous Integration service -TRAVIS = bool(os.environ.get('TRAVIS')) -APPVEYOR = bool(os.environ.get('APPVEYOR')) -CIRRUS = bool(os.environ.get('CIRRUS')) -CI_TESTING = TRAVIS or APPVEYOR or CIRRUS +APPVEYOR = 'APPVEYOR' in os.environ +GITHUB_ACTIONS = 'GITHUB_ACTIONS' in os.environ or 'CIBUILDWHEEL' in os.environ +CI_TESTING = APPVEYOR or GITHUB_ACTIONS +# are we a 64 bit process? +IS_64BIT = sys.maxsize > 2 ** 32 + # --- configurable defaults # how many times retry_on_failure() decorator will retry NO_RETRIES = 10 -# bytes tolerance for system-wide memory related tests -MEMORY_TOLERANCE = 500 * 1024 # 500KB +# bytes tolerance for system-wide related tests +TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB +TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB # the timeout used in functions which have to wait GLOBAL_TIMEOUT = 5 -# be more tolerant if we're on travis / appveyor in order to avoid -# false positives -if TRAVIS or APPVEYOR: +# be more tolerant if we're on CI in order to avoid false positives +if CI_TESTING: NO_RETRIES *= 3 GLOBAL_TIMEOUT *= 3 + TOLERANCE_SYS_MEM *= 3 + TOLERANCE_DISK_USAGE *= 3 -# --- files +# --- file names -TESTFILE_PREFIX = '$testfn' +# Disambiguate TESTFN for parallel testing. if os.name == 'java': # Jython disallows @ in module names - TESTFILE_PREFIX = '$psutil-test-' + TESTFN_PREFIX = '$psutil-%s-' % os.getpid() +else: + TESTFN_PREFIX = '@psutil-%s-' % os.getpid() +UNICODE_SUFFIX = u("-ƒőő") +# An invalid unicode string. +if PY3: + INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape') else: - TESTFILE_PREFIX = '@psutil-test-' -TESTFN = os.path.join(os.path.realpath(os.getcwd()), TESTFILE_PREFIX) -# Disambiguate TESTFN for parallel testing, while letting it remain a valid -# module name. -TESTFN = TESTFN + str(os.getpid()) - -_TESTFN = TESTFN + '-internal' -TESTFN_UNICODE = TESTFN + u("-ƒőő") + INVALID_UNICODE_SUFFIX = "f\xc0\x80" ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii') # --- paths @@ -175,7 +187,7 @@ HAS_DISK_SWAPS = hasattr(psutil, "disk_swaps") try: HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery()) except Exception: - HAS_BATTERY = True + HAS_BATTERY = False HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") HAS_THREADS = hasattr(psutil.Process, "threads") @@ -194,7 +206,14 @@ def _get_py_exe(): else: return exe - if MACOS: + if GITHUB_ACTIONS: + if PYPY: + return which("pypy3") if PY3 else which("pypy") + elif FREEBSD: + return os.path.realpath(sys.executable) + else: + return which('python') + elif MACOS: exe = \ attempt(sys.executable) or \ attempt(os.path.realpath(sys.executable)) or \ @@ -211,39 +230,14 @@ def _get_py_exe(): PYTHON_EXE = _get_py_exe() DEVNULL = open(os.devnull, 'r+') +atexit.register(DEVNULL.close) + VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] AF_UNIX = getattr(socket, "AF_UNIX", object()) _subprocesses_started = set() _pids_started = set() -_testfiles_created = set() - - -@atexit.register -def cleanup_test_files(): - DEVNULL.close() - for name in os.listdir(u('.')): - if isinstance(name, unicode): - prefix = u(TESTFILE_PREFIX) - else: - prefix = TESTFILE_PREFIX - if name.startswith(prefix): - try: - safe_rmpath(name) - except Exception: - traceback.print_exc() - for path in _testfiles_created: - try: - safe_rmpath(path) - except Exception: - traceback.print_exc() - - -# this is executed first -@atexit.register -def cleanup_test_procs(): - reap_children(recursive=True) # =================================================================== @@ -255,7 +249,7 @@ class ThreadTask(threading.Thread): """A thread task which does nothing expect staying alive.""" def __init__(self): - threading.Thread.__init__(self) + super().__init__() self._running = False self._interval = 0.001 self._flag = threading.Event() @@ -311,9 +305,9 @@ def _reap_children_on_err(fun): @_reap_children_on_err -def get_test_subprocess(cmd=None, **kwds): +def spawn_testproc(cmd=None, **kwds): """Creates a python subprocess which does nothing for 60 secs and - return it as subprocess.Popen instance. + return it as a subprocess.Popen instance. If "cmd" is specified that is used instead of python. By default stdin and stdout are redirected to /dev/null. It also attemps to make sure the process is in a reasonably @@ -331,14 +325,18 @@ def get_test_subprocess(cmd=None, **kwds): CREATE_NO_WINDOW = 0x8000000 kwds.setdefault("creationflags", CREATE_NO_WINDOW) if cmd is None: - safe_rmpath(_TESTFN) - pyline = "from time import sleep;" \ - "open(r'%s', 'w').close();" \ - "sleep(60);" % _TESTFN - cmd = [PYTHON_EXE, "-c", pyline] - sproc = subprocess.Popen(cmd, **kwds) - _subprocesses_started.add(sproc) - wait_for_file(_TESTFN, delete=True, empty=True) + testfn = get_testfn() + try: + safe_rmpath(testfn) + pyline = "from time import sleep;" \ + "open(r'%s', 'w').close();" \ + "sleep(60);" % testfn + cmd = [PYTHON_EXE, "-c", pyline] + sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) + wait_for_file(testfn, delete=True, empty=True) + finally: + safe_rmpath(testfn) else: sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) @@ -347,44 +345,51 @@ def get_test_subprocess(cmd=None, **kwds): @_reap_children_on_err -def create_proc_children_pair(): +def spawn_children_pair(): """Create a subprocess which creates another one as in: A (us) -> B (child) -> C (grandchild). Return a (child, grandchild) tuple. The 2 processes are fully initialized and will live for 60 secs and are registered for cleanup on reap_children(). """ - _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative - s = textwrap.dedent("""\ - import subprocess, os, sys, time - s = "import os, time;" - s += "f = open('%s', 'w');" - s += "f.write(str(os.getpid()));" - s += "f.close();" - s += "time.sleep(60);" - p = subprocess.Popen([r'%s', '-c', s]) - p.wait() - """ % (_TESTFN2, PYTHON_EXE)) - # On Windows if we create a subprocess with CREATE_NO_WINDOW flag - # set (which is the default) a "conhost.exe" extra process will be - # spawned as a child. We don't want that. - if WINDOWS: - subp = pyrun(s, creationflags=0) - else: - subp = pyrun(s) - child1 = psutil.Process(subp.pid) - data = wait_for_file(_TESTFN2, delete=False, empty=False) - safe_rmpath(_TESTFN2) - child2_pid = int(data) - _pids_started.add(child2_pid) - child2 = psutil.Process(child2_pid) - return (child1, child2) - - -def create_zombie_proc(): - """Create a zombie process and return its PID.""" + tfile = None + testfn = get_testfn(dir=os.getcwd()) + try: + s = textwrap.dedent("""\ + import subprocess, os, sys, time + s = "import os, time;" + s += "f = open('%s', 'w');" + s += "f.write(str(os.getpid()));" + s += "f.close();" + s += "time.sleep(60);" + p = subprocess.Popen([r'%s', '-c', s]) + p.wait() + """ % (os.path.basename(testfn), PYTHON_EXE)) + # On Windows if we create a subprocess with CREATE_NO_WINDOW flag + # set (which is the default) a "conhost.exe" extra process will be + # spawned as a child. We don't want that. + if WINDOWS: + subp, tfile = pyrun(s, creationflags=0) + else: + subp, tfile = pyrun(s) + child = psutil.Process(subp.pid) + grandchild_pid = int(wait_for_file(testfn, delete=True, empty=False)) + _pids_started.add(grandchild_pid) + grandchild = psutil.Process(grandchild_pid) + return (child, grandchild) + finally: + safe_rmpath(testfn) + if tfile is not None: + safe_rmpath(tfile) + + +def spawn_zombie(): + """Create a zombie process and return a (parent, zombie) process tuple. + In order to kill the zombie parent must be terminate()d first, then + zombie must be wait()ed on. + """ assert psutil.POSIX - unix_file = tempfile.mktemp(prefix=TESTFILE_PREFIX) if MACOS else TESTFN + unix_file = get_testfn() src = textwrap.dedent("""\ import os, sys, time, socket, contextlib child_pid = os.fork() @@ -401,38 +406,46 @@ def create_zombie_proc(): pid = bytes(str(os.getpid()), 'ascii') s.sendall(pid) """ % unix_file) - with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: + tfile = None + sock = bind_unix_socket(unix_file) + try: sock.settimeout(GLOBAL_TIMEOUT) - sock.bind(unix_file) - sock.listen(5) - pyrun(src) + parent, tfile = pyrun(src) conn, _ = sock.accept() try: select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) zpid = int(conn.recv(1024)) _pids_started.add(zpid) - zproc = psutil.Process(zpid) - call_until(lambda: zproc.status(), "ret == psutil.STATUS_ZOMBIE") - return zpid + zombie = psutil.Process(zpid) + call_until(lambda: zombie.status(), "ret == psutil.STATUS_ZOMBIE") + return (parent, zombie) finally: conn.close() + finally: + sock.close() + safe_rmpath(unix_file) + if tfile is not None: + safe_rmpath(tfile) @_reap_children_on_err def pyrun(src, **kwds): """Run python 'src' code string in a separate interpreter. - Returns a subprocess.Popen instance. + Returns a subprocess.Popen instance and the test file where the source + code was written. """ kwds.setdefault("stdout", None) kwds.setdefault("stderr", None) - with tempfile.NamedTemporaryFile( - prefix=TESTFILE_PREFIX, mode="wt", delete=False) as f: - _testfiles_created.add(f.name) - f.write(src) - f.flush() - subp = get_test_subprocess([PYTHON_EXE, f.name], **kwds) + srcfile = get_testfn() + try: + with open(srcfile, 'wt') as f: + f.write(src) + subp = spawn_testproc([PYTHON_EXE, f.name], **kwds) wait_for_pid(subp.pid) - return subp + return (subp, srcfile) + except Exception: + safe_rmpath(srcfile) + raise @_reap_children_on_err @@ -463,95 +476,121 @@ def sh(cmd, **kwds): return stdout -def reap_children(recursive=False): - """Terminate and wait() any subprocess started by this test suite - and ensure that no zombies stick around to hog resources and - create problems when looking for refleaks. - - If resursive is True it also tries to terminate and wait() - all grandchildren started by this process. +def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT): + """Terminate a process and wait() for it. + Process can be a PID or an instance of psutil.Process(), + subprocess.Popen() or psutil.Popen(). + If it's a subprocess.Popen() or psutil.Popen() instance also closes + its stdin / stdout / stderr fds. + PID is wait()ed even if the process is already gone (kills zombies). + Does nothing if the process does not exist. + Return process exit status. """ - # This is here to make sure wait_procs() behaves properly and - # investigate: - # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ - # jiq2cgd6stsbtn60 - def assert_gone(pid): - assert not psutil.pid_exists(pid), pid - assert pid not in psutil.pids(), pid - try: - p = psutil.Process(pid) - assert not p.is_running(), pid - except psutil.NoSuchProcess: - pass + def wait(proc, timeout): + if isinstance(proc, subprocess.Popen) and not PY3: + proc.wait() else: - assert 0, "pid %s is not gone" % pid - - # Get the children here, before terminating the children sub - # processes as we don't want to lose the intermediate reference - # in case of grandchildren. - if recursive: - children = set(psutil.Process().children(recursive=True)) - else: - children = set() + proc.wait(timeout) + if WINDOWS and isinstance(proc, subprocess.Popen): + # Otherwise PID may still hang around. + try: + return psutil.Process(proc.pid).wait(timeout) + except psutil.NoSuchProcess: + pass - # Terminate subprocess.Popen instances "cleanly" by closing their - # fds and wiat()ing for them in order to avoid zombies. - while _subprocesses_started: - subp = _subprocesses_started.pop() - _pids_started.add(subp.pid) + def sendsig(proc, sig): + # XXX: otherwise the build hangs for some reason. + if MACOS and GITHUB_ACTIONS: + sig = signal.SIGKILL + # If the process received SIGSTOP, SIGCONT is necessary first, + # otherwise SIGTERM won't work. + if POSIX and sig != signal.SIGKILL: + proc.send_signal(signal.SIGCONT) + proc.send_signal(sig) + + def term_subproc(proc, timeout): try: - subp.terminate() + sendsig(proc, sig) except OSError as err: if WINDOWS and err.winerror == 6: # "invalid handle" pass elif err.errno != errno.ESRCH: raise - if subp.stdout: - subp.stdout.close() - if subp.stderr: - subp.stderr.close() + return wait(proc, timeout) + + def term_psproc(proc, timeout): try: - # Flushing a BufferedWriter may raise an error. - if subp.stdin: - subp.stdin.close() - finally: - # Wait for the process to terminate, to avoid zombies. - try: - subp.wait() - except ChildProcessError: - pass + sendsig(proc, sig) + except psutil.NoSuchProcess: + pass + return wait(proc, timeout) - # Terminate started pids. - while _pids_started: - pid = _pids_started.pop() + def term_pid(pid, timeout): try: - p = psutil.Process(pid) + proc = psutil.Process(pid) except psutil.NoSuchProcess: - assert_gone(pid) + # Needed to kill zombies. + if POSIX: + return wait_pid(pid, timeout) + else: + return term_psproc(proc, timeout) + + def flush_popen(proc): + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() + # Flushing a BufferedWriter may raise an error. + if proc.stdin: + proc.stdin.close() + + p = proc_or_pid + try: + if isinstance(p, int): + return term_pid(p, wait_timeout) + elif isinstance(p, (psutil.Process, psutil.Popen)): + return term_psproc(p, wait_timeout) + elif isinstance(p, subprocess.Popen): + return term_subproc(p, wait_timeout) else: - children.add(p) + raise TypeError("wrong type %r" % p) + finally: + if isinstance(p, (subprocess.Popen, psutil.Popen)): + flush_popen(p) + pid = p if isinstance(p, int) else p.pid + assert not psutil.pid_exists(pid), pid + + +def reap_children(recursive=False): + """Terminate and wait() any subprocess started by this test suite + and any children currently running, ensuring that no processes stick + around to hog resources. + If resursive is True it also tries to terminate and wait() + all grandchildren started by this process. + """ + # Get the children here before terminating them, as in case of + # recursive=True we don't want to lose the intermediate reference + # pointing to the grandchildren. + children = psutil.Process().children(recursive=recursive) + + # Terminate subprocess.Popen. + while _subprocesses_started: + subp = _subprocesses_started.pop() + terminate(subp) + + # Collect started pids. + while _pids_started: + pid = _pids_started.pop() + terminate(pid) # Terminate children. if children: for p in children: - try: - p.terminate() - except psutil.NoSuchProcess: - pass + terminate(p, wait_timeout=None) gone, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT) for p in alive: warn("couldn't terminate process %r; attempting kill()" % p) - try: - p.kill() - except psutil.NoSuchProcess: - pass - gone, alive = psutil.wait_procs(alive, timeout=GLOBAL_TIMEOUT) - if alive: - for p in alive: - warn("process %r survived kill()" % p) - - for p in children: - assert_gone(p.pid) + terminate(p, sig=signal.SIGKILL) # =================================================================== @@ -559,7 +598,7 @@ def reap_children(recursive=False): # =================================================================== -def get_kernel_version(): +def kernel_version(): """Return a tuple such as (2, 6, 36).""" if not POSIX: raise NotImplementedError("not POSIX") @@ -673,7 +712,7 @@ def wait_for_pid(pid): time.sleep(0.01) -@retry(exception=(EnvironmentError, AssertionError), logfun=None, +@retry(exception=(FileNotFoundError, AssertionError), logfun=None, timeout=GLOBAL_TIMEOUT, interval=0.001) def wait_for_file(fname, delete=True, empty=False): """Wait for a file to be written on disk with some content.""" @@ -709,7 +748,7 @@ def safe_rmpath(path): # open handles or references preventing the delete operation # to succeed immediately, so we retry for a while. See: # https://bugs.python.org/issue33240 - stop_at = time.time() + 1 + stop_at = time.time() + GLOBAL_TIMEOUT while time.time() < stop_at: try: return fun() @@ -770,8 +809,7 @@ def create_exe(outpath, c_code=None): } """) assert isinstance(c_code, str), c_code - with tempfile.NamedTemporaryFile( - suffix='.c', delete=False, mode='wt') as f: + with open(get_testfn(suffix='.c'), 'wt') as f: f.write(c_code) try: subprocess.check_call(["gcc", f.name, "-o", outpath]) @@ -785,8 +823,16 @@ def create_exe(outpath, c_code=None): os.chmod(outpath, st.st_mode | stat.S_IEXEC) -def unique_filename(prefix=TESTFILE_PREFIX, suffix=""): - return tempfile.mktemp(prefix=prefix, suffix=suffix) +def get_testfn(suffix="", dir=None): + """Return an absolute pathname of a file or dir that did not + exist at the time this call is made. Also schedule it for safe + deletion at interpreter exit. It's technically racy but probably + not really due to the time variant. + """ + while True: + name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir) + if not os.path.exists(name): # also include dirs + return os.path.realpath(name) # needed for OSX # =================================================================== @@ -810,17 +856,540 @@ class TestCase(unittest.TestCase): if not hasattr(unittest.TestCase, 'assertRaisesRegex'): assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + # ...otherwise multiprocessing.Pool complains + if not PY3: + def runTest(self): + pass + -# override default unittest.TestCase +# monkey patch default unittest.TestCase unittest.TestCase = TestCase +class PsutilTestCase(TestCase): + """Test class providing auto-cleanup wrappers on top of process + test utilities. + """ + + def get_testfn(self, suffix="", dir=None): + fname = get_testfn(suffix=suffix, dir=dir) + self.addCleanup(safe_rmpath, fname) + return fname + + def spawn_testproc(self, *args, **kwds): + sproc = spawn_testproc(*args, **kwds) + self.addCleanup(terminate, sproc) + return sproc + + def spawn_children_pair(self): + child1, child2 = spawn_children_pair() + self.addCleanup(terminate, child2) + self.addCleanup(terminate, child1) # executed first + return (child1, child2) + + def spawn_zombie(self): + parent, zombie = spawn_zombie() + self.addCleanup(terminate, zombie) + self.addCleanup(terminate, parent) # executed first + return (parent, zombie) + + def pyrun(self, *args, **kwds): + sproc, srcfile = pyrun(*args, **kwds) + self.addCleanup(safe_rmpath, srcfile) + self.addCleanup(terminate, sproc) # executed first + return sproc + + def assertProcessGone(self, proc): + self.assertRaises(psutil.NoSuchProcess, psutil.Process, proc.pid) + if isinstance(proc, (psutil.Process, psutil.Popen)): + assert not proc.is_running() + try: + status = proc.status() + except psutil.NoSuchProcess: + pass + else: + raise AssertionError("Process.status() didn't raise exception " + "(status=%s)" % status) + proc.wait(timeout=0) # assert not raise TimeoutExpired + assert not psutil.pid_exists(proc.pid), proc.pid + self.assertNotIn(proc.pid, psutil.pids()) + + +@unittest.skipIf(PYPY, "unreliable on PYPY") +class TestMemoryLeak(PsutilTestCase): + """Test framework class for detecting function memory leaks, + typically functions implemented in C which forgot to free() memory + from the heap. It does so by checking whether the process memory + usage increased before and after calling the function many times. + + Note that this is hard (probably impossible) to do reliably, due + to how the OS handles memory, the GC and so on (memory can even + decrease!). In order to avoid false positives, in case of failure + (mem > 0) we retry the test for up to 5 times, increasing call + repetitions each time. If the memory keeps increasing then it's a + failure. + + If available (Linux, OSX, Windows), USS memory is used for comparison, + since it's supposed to be more precise, see: + https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python + If not, RSS memory is used. mallinfo() on Linux and _heapwalk() on + Windows may give even more precision, but at the moment are not + implemented. + + PyPy appears to be completely unstable for this framework, probably + because of its JIT, so tests on PYPY are skipped. + + Usage: + + class TestLeaks(psutil.tests.TestMemoryLeak): + + def test_fun(self): + self.execute(some_function) + """ + # Configurable class attrs. + times = 200 + warmup_times = 10 + tolerance = 0 # memory + retries = 10 if CI_TESTING else 5 + verbose = True + _thisproc = psutil.Process() + _psutil_debug_orig = bool(os.getenv('PSUTIL_DEBUG', 0)) + + @classmethod + def setUpClass(cls): + psutil._set_debug(False) # avoid spamming to stderr + + @classmethod + def tearDownClass(cls): + psutil._set_debug(cls._psutil_debug_orig) + + def _get_mem(self): + # USS is the closest thing we have to "real" memory usage and it + # should be less likely to produce false positives. + mem = self._thisproc.memory_full_info() + return getattr(mem, "uss", mem.rss) + + def _get_num_fds(self): + if POSIX: + return self._thisproc.num_fds() + else: + return self._thisproc.num_handles() + + def _log(self, msg): + if self.verbose: + print_color(msg, color="yellow", file=sys.stderr) + + def _check_fds(self, fun): + """Makes sure num_fds() (POSIX) or num_handles() (Windows) does + not increase after calling a function. Used to discover forgotten + close(2) and CloseHandle syscalls. + """ + before = self._get_num_fds() + self.call(fun) + after = self._get_num_fds() + diff = after - before + if diff < 0: + raise self.fail("negative diff %r (gc probably collected a " + "resource from a previous test)" % diff) + if diff > 0: + type_ = "fd" if POSIX else "handle" + if diff > 1: + type_ += "s" + msg = "%s unclosed %s after calling %r" % (diff, type_, fun) + raise self.fail(msg) + + def _call_ntimes(self, fun, times): + """Get 2 distinct memory samples, before and after having + called fun repeadetly, and return the memory difference. + """ + gc.collect(generation=1) + mem1 = self._get_mem() + for x in range(times): + ret = self.call(fun) + del x, ret + gc.collect(generation=1) + mem2 = self._get_mem() + self.assertEqual(gc.garbage, []) + diff = mem2 - mem1 # can also be negative + return diff + + def _check_mem(self, fun, times, warmup_times, retries, tolerance): + messages = [] + prev_mem = 0 + increase = times + for idx in range(1, retries + 1): + mem = self._call_ntimes(fun, times) + msg = "Run #%s: extra-mem=%s, per-call=%s, calls=%s" % ( + idx, bytes2human(mem), bytes2human(mem / times), times) + messages.append(msg) + success = mem <= tolerance or mem <= prev_mem + if success: + if idx > 1: + self._log(msg) + return + else: + if idx == 1: + print() # NOQA + self._log(msg) + times += increase + prev_mem = mem + raise self.fail(". ".join(messages)) + + # --- + + def call(self, fun): + return fun() + + def execute(self, fun, times=None, warmup_times=None, retries=None, + tolerance=None): + """Test a callable.""" + times = times if times is not None else self.times + warmup_times = warmup_times if warmup_times is not None \ + else self.warmup_times + retries = retries if retries is not None else self.retries + tolerance = tolerance if tolerance is not None else self.tolerance + try: + assert times >= 1, "times must be >= 1" + assert warmup_times >= 0, "warmup_times must be >= 0" + assert retries >= 0, "retries must be >= 0" + assert tolerance >= 0, "tolerance must be >= 0" + except AssertionError as err: + raise ValueError(str(err)) + + self._call_ntimes(fun, warmup_times) # warm up + self._check_fds(fun) + self._check_mem(fun, times=times, warmup_times=warmup_times, + retries=retries, tolerance=tolerance) + + def execute_w_exc(self, exc, fun, **kwargs): + """Convenience method to test a callable while making sure it + raises an exception on every call. + """ + def call(): + self.assertRaises(exc, fun) + + self.execute(call, **kwargs) + + +def print_sysinfo(): + import collections + import datetime + import getpass + import locale + import platform + import pprint + try: + import pip + except ImportError: + pip = None + try: + import wheel + except ImportError: + wheel = None + + info = collections.OrderedDict() + + # OS + if psutil.LINUX and which('lsb_release'): + info['OS'] = sh('lsb_release -d -s') + elif psutil.OSX: + info['OS'] = 'Darwin %s' % platform.mac_ver()[0] + elif psutil.WINDOWS: + info['OS'] = "Windows " + ' '.join( + map(str, platform.win32_ver())) + if hasattr(platform, 'win32_edition'): + info['OS'] += ", " + platform.win32_edition() + else: + info['OS'] = "%s %s" % (platform.system(), platform.version()) + info['arch'] = ', '.join( + list(platform.architecture()) + [platform.machine()]) + if psutil.POSIX: + info['kernel'] = platform.uname()[2] + + # python + info['python'] = ', '.join([ + platform.python_implementation(), + platform.python_version(), + platform.python_compiler()]) + info['pip'] = getattr(pip, '__version__', 'not installed') + if wheel is not None: + info['pip'] += " (wheel=%s)" % wheel.__version__ + + # UNIX + if psutil.POSIX: + if which('gcc'): + out = sh(['gcc', '--version']) + info['gcc'] = str(out).split('\n')[0] + else: + info['gcc'] = 'not installed' + s = platform.libc_ver()[1] + if s: + info['glibc'] = s + + # system + info['fs-encoding'] = sys.getfilesystemencoding() + lang = locale.getlocale() + info['lang'] = '%s, %s' % (lang[0], lang[1]) + info['boot-time'] = datetime.datetime.fromtimestamp( + psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S") + info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + info['user'] = getpass.getuser() + info['home'] = os.path.expanduser("~") + info['cwd'] = os.getcwd() + info['pyexe'] = PYTHON_EXE + info['hostname'] = platform.node() + info['PID'] = os.getpid() + + # metrics + info['cpus'] = psutil.cpu_count() + info['loadavg'] = "%.1f%%, %.1f%%, %.1f%%" % ( + tuple([x / psutil.cpu_count() * 100 for x in psutil.getloadavg()])) + mem = psutil.virtual_memory() + info['memory'] = "%s%%, used=%s, total=%s" % ( + int(mem.percent), bytes2human(mem.used), bytes2human(mem.total)) + swap = psutil.swap_memory() + info['swap'] = "%s%%, used=%s, total=%s" % ( + int(swap.percent), bytes2human(swap.used), bytes2human(swap.total)) + info['pids'] = len(psutil.pids()) + pinfo = psutil.Process().as_dict() + pinfo.pop('memory_maps', None) + info['proc'] = pprint.pformat(pinfo) + + print("=" * 70, file=sys.stderr) # NOQA + for k, v in info.items(): + print("%-17s %s" % (k + ':', v), file=sys.stderr) # NOQA + print("=" * 70, file=sys.stderr) # NOQA + sys.stdout.flush() + + +def _get_eligible_cpu(): + p = psutil.Process() + if hasattr(p, "cpu_num"): + return p.cpu_num() + elif hasattr(p, "cpu_affinity"): + return random.choice(p.cpu_affinity()) + return 0 + + +class process_namespace: + """A container that lists all Process class method names + some + reasonable parameters to be called with. Utility methods (parent(), + children(), ...) are excluded. + + >>> ns = process_namespace(psutil.Process()) + >>> for fun, name in ns.iter(ns.getters): + ... fun() + """ + utils = [ + ('cpu_percent', (), {}), + ('memory_percent', (), {}), + ] + + ignored = [ + ('as_dict', (), {}), + ('children', (), {'recursive': True}), + ('is_running', (), {}), + ('memory_info_ex', (), {}), + ('oneshot', (), {}), + ('parent', (), {}), + ('parents', (), {}), + ('pid', (), {}), + ('wait', (0, ), {}), + ] + + getters = [ + ('cmdline', (), {}), + ('connections', (), {'kind': 'all'}), + ('cpu_times', (), {}), + ('create_time', (), {}), + ('cwd', (), {}), + ('exe', (), {}), + ('memory_full_info', (), {}), + ('memory_info', (), {}), + ('name', (), {}), + ('nice', (), {}), + ('num_ctx_switches', (), {}), + ('num_threads', (), {}), + ('open_files', (), {}), + ('ppid', (), {}), + ('status', (), {}), + ('threads', (), {}), + ('username', (), {}), + ] + if POSIX: + getters += [('uids', (), {})] + getters += [('gids', (), {})] + getters += [('terminal', (), {})] + getters += [('num_fds', (), {})] + if HAS_PROC_IO_COUNTERS: + getters += [('io_counters', (), {})] + if HAS_IONICE: + getters += [('ionice', (), {})] + if HAS_RLIMIT: + getters += [('rlimit', (psutil.RLIMIT_NOFILE, ), {})] + if HAS_CPU_AFFINITY: + getters += [('cpu_affinity', (), {})] + if HAS_PROC_CPU_NUM: + getters += [('cpu_num', (), {})] + if HAS_ENVIRON: + getters += [('environ', (), {})] + if WINDOWS: + getters += [('num_handles', (), {})] + if HAS_MEMORY_MAPS: + getters += [('memory_maps', (), {'grouped': False})] + + setters = [] + if POSIX: + setters += [('nice', (0, ), {})] + else: + setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS, ), {})] + if HAS_RLIMIT: + setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})] + if HAS_IONICE: + if LINUX: + setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})] + else: + setters += [('ionice', (psutil.IOPRIO_NORMAL, ), {})] + if HAS_CPU_AFFINITY: + setters += [('cpu_affinity', ([_get_eligible_cpu()], ), {})] + + killers = [ + ('send_signal', (signal.SIGTERM, ), {}), + ('suspend', (), {}), + ('resume', (), {}), + ('terminate', (), {}), + ('kill', (), {}), + ] + if WINDOWS: + killers += [('send_signal', (signal.CTRL_C_EVENT, ), {})] + killers += [('send_signal', (signal.CTRL_BREAK_EVENT, ), {})] + + all = utils + getters + setters + killers + + def __init__(self, proc): + self._proc = proc + + def iter(self, ls, clear_cache=True): + """Given a list of tuples yields a set of (fun, fun_name) tuples + in random order. + """ + ls = list(ls) + random.shuffle(ls) + for fun_name, args, kwds in ls: + if clear_cache: + self.clear_cache() + fun = getattr(self._proc, fun_name) + fun = functools.partial(fun, *args, **kwds) + yield (fun, fun_name) + + def clear_cache(self): + """Clear the cache of a Process instance.""" + self._proc._init(self._proc.pid, _ignore_nsp=True) + + @classmethod + def test_class_coverage(cls, test_class, ls): + """Given a TestCase instance and a list of tuples checks that + the class defines the required test method names. + """ + for fun_name, _, _ in ls: + meth_name = 'test_' + fun_name + if not hasattr(test_class, meth_name): + msg = "%r class should define a '%s' method" % ( + test_class.__class__.__name__, meth_name) + raise AttributeError(msg) + + @classmethod + def test(cls): + this = set([x[0] for x in cls.all]) + ignored = set([x[0] for x in cls.ignored]) + klass = set([x for x in dir(psutil.Process) if x[0] != '_']) + leftout = (this | ignored) ^ klass + if leftout: + raise ValueError("uncovered Process class names: %r" % leftout) + + +class system_namespace: + """A container that lists all the module-level, system-related APIs. + Utilities such as cpu_percent() are excluded. Usage: + + >>> ns = system_namespace + >>> for fun, name in ns.iter(ns.getters): + ... fun() + """ + getters = [ + ('boot_time', (), {}), + ('cpu_count', (), {'logical': False}), + ('cpu_count', (), {'logical': True}), + ('cpu_stats', (), {}), + ('cpu_times', (), {'percpu': False}), + ('cpu_times', (), {'percpu': True}), + ('disk_io_counters', (), {'perdisk': True}), + ('disk_partitions', (), {'all': True}), + ('disk_usage', (os.getcwd(), ), {}), + ('net_connections', (), {'kind': 'all'}), + ('net_if_addrs', (), {}), + ('net_if_stats', (), {}), + ('net_io_counters', (), {'pernic': True}), + ('pid_exists', (os.getpid(), ), {}), + ('pids', (), {}), + ('swap_memory', (), {}), + ('users', (), {}), + ('virtual_memory', (), {}), + ] + if HAS_CPU_FREQ: + getters += [('cpu_freq', (), {'percpu': True})] + if HAS_GETLOADAVG: + getters += [('getloadavg', (), {})] + if HAS_SENSORS_TEMPERATURES: + getters += [('sensors_temperatures', (), {})] + if HAS_SENSORS_FANS: + getters += [('sensors_fans', (), {})] + if HAS_SENSORS_BATTERY: + getters += [('sensors_battery', (), {})] + if WINDOWS: + getters += [('win_service_iter', (), {})] + getters += [('win_service_get', ('alg', ), {})] + + ignored = [ + ('process_iter', (), {}), + ('wait_procs', ([psutil.Process()], ), {}), + ('cpu_percent', (), {}), + ('cpu_times_percent', (), {}), + ] + + all = getters + + @staticmethod + def iter(ls): + """Given a list of tuples yields a set of (fun, fun_name) tuples + in random order. + """ + ls = list(ls) + random.shuffle(ls) + for fun_name, args, kwds in ls: + fun = getattr(psutil, fun_name) + fun = functools.partial(fun, *args, **kwds) + yield (fun, fun_name) + + test_class_coverage = process_namespace.test_class_coverage + + +def serialrun(klass): + """A decorator to mark a TestCase class. When running parallel tests, + class' unit tests will be run serially (1 process). + """ + # assert issubclass(klass, unittest.TestCase), klass + assert inspect.isclass(klass), klass + klass._serialrun = True + return klass + + def retry_on_failure(retries=NO_RETRIES): """Decorator which runs a test function and retries N times before actually failing. """ def logfun(exc): - print("%r, retrying" % exc, file=sys.stderr) + print("%r, retrying" % exc, file=sys.stderr) # NOQA return retry(exception=AssertionError, timeout=None, retries=retries, logfun=logfun) @@ -865,29 +1434,14 @@ def skip_on_not_implemented(only_if=None): # =================================================================== +# XXX: no longer used def get_free_port(host='127.0.0.1'): - """Return an unused TCP port.""" + """Return an unused TCP port. Subject to race conditions.""" with contextlib.closing(socket.socket()) as sock: sock.bind((host, 0)) return sock.getsockname()[1] -@contextlib.contextmanager -def unix_socket_path(suffix=""): - """A context manager which returns a non-existent file name - and tries to delete it on exit. - """ - assert psutil.POSIX - path = unique_filename(suffix=suffix) - try: - yield path - finally: - try: - os.unlink(path) - except OSError: - pass - - def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): """Binds a generic socket.""" if addr is None and family in (AF_INET, AF_INET6): @@ -978,22 +1532,19 @@ def create_sockets(): socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) if POSIX and HAS_CONNECTIONS_UNIX: - fname1 = unix_socket_path().__enter__() - fname2 = unix_socket_path().__enter__() + fname1 = get_testfn() + fname2 = get_testfn() s1, s2 = unix_socketpair(fname1) s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM) - # self.addCleanup(safe_rmpath, fname1) - # self.addCleanup(safe_rmpath, fname2) for s in (s1, s2, s3): socks.append(s) yield socks finally: for s in socks: s.close() - if fname1 is not None: - safe_rmpath(fname1) - if fname2 is not None: - safe_rmpath(fname2) + for fname in (fname1, fname2): + if fname is not None: + safe_rmpath(fname) def check_net_address(addr, family): @@ -1001,7 +1552,7 @@ def check_net_address(addr, family): IPv6 and MAC addresses. """ import ipaddress # python >= 3.3 / requires "pip install ipaddress" - if enum and PY3: + if enum and PY3 and not PYPY: assert isinstance(family, enum.IntEnum), family if family == socket.AF_INET: octs = [int(x) for x in addr.split('.')] @@ -1022,6 +1573,83 @@ def check_net_address(addr, family): raise ValueError("unknown family %r", family) +def check_connection_ntuple(conn): + """Check validity of a connection namedtuple.""" + def check_ntuple(conn): + has_pid = len(conn) == 7 + assert len(conn) in (6, 7), len(conn) + assert conn[0] == conn.fd, conn.fd + assert conn[1] == conn.family, conn.family + assert conn[2] == conn.type, conn.type + assert conn[3] == conn.laddr, conn.laddr + assert conn[4] == conn.raddr, conn.raddr + assert conn[5] == conn.status, conn.status + if has_pid: + assert conn[6] == conn.pid, conn.pid + + def check_family(conn): + assert conn.family in (AF_INET, AF_INET6, AF_UNIX), conn.family + if enum is not None: + assert isinstance(conn.family, enum.IntEnum), conn + else: + assert isinstance(conn.family, int), conn + if conn.family == AF_INET: + # actually try to bind the local socket; ignore IPv6 + # sockets as their address might be represented as + # an IPv4-mapped-address (e.g. "::127.0.0.1") + # and that's rejected by bind() + s = socket.socket(conn.family, conn.type) + with contextlib.closing(s): + try: + s.bind((conn.laddr[0], 0)) + except socket.error as err: + if err.errno != errno.EADDRNOTAVAIL: + raise + elif conn.family == AF_UNIX: + assert conn.status == psutil.CONN_NONE, conn.status + + def check_type(conn): + # SOCK_SEQPACKET may happen in case of AF_UNIX socks + SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) + assert conn.type in (socket.SOCK_STREAM, socket.SOCK_DGRAM, + SOCK_SEQPACKET), conn.type + if enum is not None: + assert isinstance(conn.type, enum.IntEnum), conn + else: + assert isinstance(conn.type, int), conn + if conn.type == socket.SOCK_DGRAM: + assert conn.status == psutil.CONN_NONE, conn.status + + def check_addrs(conn): + # check IP address and port sanity + for addr in (conn.laddr, conn.raddr): + if conn.family in (AF_INET, AF_INET6): + assert isinstance(addr, tuple), type(addr) + if not addr: + continue + assert isinstance(addr.port, int), type(addr.port) + assert 0 <= addr.port <= 65535, addr.port + check_net_address(addr.ip, conn.family) + elif conn.family == AF_UNIX: + assert isinstance(addr, str), type(addr) + + def check_status(conn): + assert isinstance(conn.status, str), conn.status + valids = [getattr(psutil, x) for x in dir(psutil) + if x.startswith('CONN_')] + assert conn.status in valids, conn.status + if conn.family in (AF_INET, AF_INET6) and conn.type == SOCK_STREAM: + assert conn.status != psutil.CONN_NONE, conn.status + else: + assert conn.status == psutil.CONN_NONE, conn.status + + check_ntuple(conn) + check_family(conn) + check_type(conn) + check_addrs(conn) + check_status(conn) + + # =================================================================== # --- compatibility # =================================================================== @@ -1080,14 +1708,14 @@ def is_namedtuple(x): if POSIX: @contextlib.contextmanager - def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): + def copyload_shared_lib(suffix=""): """Ctx manager which picks up a random shared CO lib used by this process, copies it in another location and loads it in memory via ctypes. Return the new absolutized path. """ exe = 'pypy' if PYPY else 'python' ext = ".so" - dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) + dst = get_testfn(suffix=suffix + ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.splitext(x.path)[1] == ext and exe in x.path.lower()] @@ -1100,7 +1728,7 @@ if POSIX: safe_rmpath(dst) else: @contextlib.contextmanager - def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): + def copyload_shared_lib(suffix=""): """Ctx manager which picks up a random shared DLL lib used by this process, copies it in another location and loads it in memory via ctypes. @@ -1109,7 +1737,7 @@ else: from ctypes import wintypes from ctypes import WinError ext = ".dll" - dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) + dst = get_testfn(suffix=suffix + ext) libs = [x.path for x in psutil.Process().memory_maps() if x.path.lower().endswith(ext) and 'python' in os.path.basename(x.path).lower() and @@ -1136,3 +1764,22 @@ else: if ret == 0: WinError() safe_rmpath(dst) + + +# =================================================================== +# --- Exit funs (first is executed last) +# =================================================================== + + +# this is executed first +@atexit.register +def cleanup_test_procs(): + reap_children(recursive=True) + + +# atexit module does not execute exit functions in case of SIGTERM, which +# gets sent to test subprocesses, which is a problem if they import this +# module. With this it will. See: +# https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python +if POSIX: + signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(sig)) diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 68710d44..d5cd02eb 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -6,89 +6,8 @@ """ Run unit tests. This is invoked by: - $ python -m psutil.tests """ -import contextlib -import optparse -import os -import sys -import tempfile -try: - from urllib.request import urlopen # py3 -except ImportError: - from urllib2 import urlopen - -from psutil.tests import PYTHON_EXE -from psutil.tests.runner import run - - -HERE = os.path.abspath(os.path.dirname(__file__)) -GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -TEST_DEPS = [] -if sys.version_info[:2] == (2, 6): - TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) -elif sys.version_info[:2] == (2, 7) or sys.version_info[:2] <= (3, 2): - TEST_DEPS.extend(["ipaddress", "mock"]) - - -def install_pip(): - try: - import pip # NOQA - except ImportError: - import ssl - f = tempfile.NamedTemporaryFile(suffix='.py') - with contextlib.closing(f): - print("downloading %s to %s" % (GET_PIP_URL, f.name)) - if hasattr(ssl, '_create_unverified_context'): - ctx = ssl._create_unverified_context() - else: - ctx = None - kwargs = dict(context=ctx) if ctx else {} - req = urlopen(GET_PIP_URL, **kwargs) - data = req.read() - f.write(data) - f.flush() - - print("installing pip") - code = os.system('%s %s --user' % (PYTHON_EXE, f.name)) - return code - - -def install_test_deps(deps=None): - """Install test dependencies via pip.""" - if deps is None: - deps = TEST_DEPS - deps = set(deps) - if deps: - is_venv = hasattr(sys, 'real_prefix') - opts = "--user" if not is_venv else "" - install_pip() - code = os.system('%s -m pip install %s --upgrade %s' % ( - PYTHON_EXE, opts, " ".join(deps))) - return code - - -def main(): - usage = "%s -m psutil.tests [opts]" % PYTHON_EXE - parser = optparse.OptionParser(usage=usage, description="run unit tests") - parser.add_option("-i", "--install-deps", - action="store_true", default=False, - help="don't print status messages to stdout") - - opts, args = parser.parse_args() - if opts.install_deps: - install_pip() - install_test_deps() - else: - for dep in TEST_DEPS: - try: - __import__(dep.split("==")[0]) - except ImportError: - sys.exit("%r lib is not installed; run %s -m psutil.tests " - "--install-deps" % (dep, PYTHON_EXE)) - run() - - +from .runner import main main() diff --git a/psutil/tests/runner.py b/psutil/tests/runner.py index f8601bad..65f222c4 100755 --- a/psutil/tests/runner.py +++ b/psutil/tests/runner.py @@ -1,197 +1,345 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ -Unit test runner, providing colourized output and printing failures -on KeyboardInterrupt. +Unit test runner, providing new features on top of unittest module: +- colourized output +- parallel run (UNIX only) +- print failures/tracebacks on CTRL+C +- re-run failed tests only (make test-failed) + +Invocation examples: +- make test +- make test-failed + +Parallel: +- make test-parallel +- make test-process ARGS=--parallel """ from __future__ import print_function import atexit +import optparse import os import sys +import textwrap +import time import unittest -from unittest import TestResult -from unittest import TextTestResult -from unittest import TextTestRunner try: import ctypes except ImportError: ctypes = None +try: + import concurrencytest # pip install concurrencytest +except ImportError: + concurrencytest = None + import psutil -from psutil._common import memoize +from psutil._common import hilite +from psutil._common import print_color +from psutil._common import term_supports_colors +from psutil._compat import super +from psutil.tests import CI_TESTING +from psutil.tests import import_module_by_path +from psutil.tests import print_sysinfo +from psutil.tests import reap_children from psutil.tests import safe_rmpath -from psutil.tests import TOX -HERE = os.path.abspath(os.path.dirname(__file__)) -VERBOSITY = 1 if TOX else 2 +VERBOSITY = 2 FAILED_TESTS_FNAME = '.failed-tests.txt' -if os.name == 'posix': - GREEN = 1 - RED = 2 - BROWN = 94 -else: - GREEN = 2 - RED = 4 - BROWN = 6 - DEFAULT_COLOR = 7 - - -def term_supports_colors(file=sys.stdout): - if os.name == 'nt': - return ctypes is not None - try: - import curses - assert file.isatty() - curses.setupterm() - assert curses.tigetnum("colors") > 0 - except Exception: - return False - else: - return True - - -def hilite(s, color, bold=False): - """Return an highlighted version of 'string'.""" - attr = [] - if color == GREEN: - attr.append('32') - elif color == RED: - attr.append('91') - elif color == BROWN: - attr.append('33') +NWORKERS = psutil.cpu_count() or 1 +USE_COLORS = not CI_TESTING and term_supports_colors() + +HERE = os.path.abspath(os.path.dirname(__file__)) +loadTestsFromTestCase = unittest.defaultTestLoader.loadTestsFromTestCase + + +def cprint(msg, color, bold=False, file=None): + if file is None: + file = sys.stderr if color == 'red' else sys.stdout + if USE_COLORS: + print_color(msg, color, bold=bold, file=file) else: - raise ValueError("unrecognized color") - if bold: - attr.append('1') - return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) - - -@memoize -def _stderr_handle(): - GetStdHandle = ctypes.windll.Kernel32.GetStdHandle - STD_ERROR_HANDLE_ID = ctypes.c_ulong(0xfffffff4) - GetStdHandle.restype = ctypes.c_ulong - handle = GetStdHandle(STD_ERROR_HANDLE_ID) - atexit.register(ctypes.windll.Kernel32.CloseHandle, handle) - return handle - - -def win_colorprint(printer, s, color, bold=False): - if bold and color <= 7: - color += 8 - handle = _stderr_handle() - SetConsoleTextAttribute = ctypes.windll.Kernel32.SetConsoleTextAttribute - SetConsoleTextAttribute(handle, color) - try: - printer(s) - finally: - SetConsoleTextAttribute(handle, DEFAULT_COLOR) - - -class ColouredResult(TextTestResult): - - def _color_print(self, s, color, bold=False): - if os.name == 'posix': - self.stream.writeln(hilite(s, color, bold=bold)) - else: - win_colorprint(self.stream.writeln, s, color, bold=bold) + print(msg, file=file) + + +class TestLoader: + + testdir = HERE + skip_files = ['test_memleaks.py'] + if "WHEELHOUSE_UPLOADER_USERNAME" in os.environ: + skip_files.extend(['test_osx.py', 'test_linux.py', 'test_posix.py']) + + def _get_testmods(self): + return [os.path.join(self.testdir, x) + for x in os.listdir(self.testdir) + if x.startswith('test_') and x.endswith('.py') and + x not in self.skip_files] + + def _iter_testmod_classes(self): + """Iterate over all test files in this directory and return + all TestCase classes in them. + """ + for path in self._get_testmods(): + mod = import_module_by_path(path) + for name in dir(mod): + obj = getattr(mod, name) + if isinstance(obj, type) and \ + issubclass(obj, unittest.TestCase): + yield obj + + def all(self): + suite = unittest.TestSuite() + for obj in self._iter_testmod_classes(): + test = loadTestsFromTestCase(obj) + suite.addTest(test) + return suite + + def last_failed(self): + # ...from previously failed test run + suite = unittest.TestSuite() + if not os.path.isfile(FAILED_TESTS_FNAME): + return suite + with open(FAILED_TESTS_FNAME, 'rt') as f: + names = f.read().split() + for n in names: + test = unittest.defaultTestLoader.loadTestsFromName(n) + suite.addTest(test) + return suite + + def from_name(self, name): + if name.endswith('.py'): + name = os.path.splitext(os.path.basename(name))[0] + return unittest.defaultTestLoader.loadTestsFromName(name) + + +class ColouredResult(unittest.TextTestResult): def addSuccess(self, test): - TestResult.addSuccess(self, test) - self._color_print("OK", GREEN) + unittest.TestResult.addSuccess(self, test) + cprint("OK", "green") def addError(self, test, err): - TestResult.addError(self, test, err) - self._color_print("ERROR", RED, bold=True) + unittest.TestResult.addError(self, test, err) + cprint("ERROR", "red", bold=True) def addFailure(self, test, err): - TestResult.addFailure(self, test, err) - self._color_print("FAIL", RED) + unittest.TestResult.addFailure(self, test, err) + cprint("FAIL", "red") def addSkip(self, test, reason): - TestResult.addSkip(self, test, reason) - self._color_print("skipped: %s" % reason, BROWN) + unittest.TestResult.addSkip(self, test, reason) + cprint("skipped: %s" % reason.strip(), "brown") def printErrorList(self, flavour, errors): - flavour = hilite(flavour, RED, bold=flavour == 'ERROR') - TextTestResult.printErrorList(self, flavour, errors) + flavour = hilite(flavour, "red", bold=flavour == 'ERROR') + super().printErrorList(flavour, errors) -class ColouredRunner(TextTestRunner): - resultclass = ColouredResult if term_supports_colors() else TextTestResult +class ColouredTextRunner(unittest.TextTestRunner): + """ + A coloured text runner which also prints failed tests on KeyboardInterrupt + and save failed tests in a file so that they can be re-run. + """ + resultclass = ColouredResult if USE_COLORS else unittest.TextTestResult + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.failed_tnames = set() def _makeResult(self): # Store result instance so that it can be accessed on # KeyboardInterrupt. - self.result = TextTestRunner._makeResult(self) + self.result = super()._makeResult() return self.result - -def setup_tests(): - if 'PSUTIL_TESTING' not in os.environ: - # This won't work on Windows but set_testing() below will do it. - os.environ['PSUTIL_TESTING'] = '1' - psutil._psplatform.cext.set_testing() - - -def get_suite(name=None): - suite = unittest.TestSuite() - if name is None: - testmods = [os.path.splitext(x)[0] for x in os.listdir(HERE) - if x.endswith('.py') and x.startswith('test_') and not - x.startswith('test_memory_leaks')] - if "WHEELHOUSE_UPLOADER_USERNAME" in os.environ: - testmods = [x for x in testmods if not x.endswith(( - "osx", "posix", "linux"))] - for tm in testmods: - # ...so that the full test paths are printed on screen - tm = "psutil.tests.%s" % tm - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) + def _write_last_failed(self): + if self.failed_tnames: + with open(FAILED_TESTS_FNAME, 'wt') as f: + for tname in self.failed_tnames: + f.write(tname + '\n') + + def _save_result(self, result): + if not result.wasSuccessful(): + for t in result.errors + result.failures: + tname = t[0].id() + self.failed_tnames.add(tname) + + def _run(self, suite): + try: + result = super().run(suite) + except (KeyboardInterrupt, SystemExit): + result = self.runner.result + result.printErrors() + raise sys.exit(1) + else: + self._save_result(result) + return result + + def _exit(self, success): + if success: + cprint("SUCCESS", "green", bold=True) + safe_rmpath(FAILED_TESTS_FNAME) + sys.exit(0) + else: + cprint("FAILED", "red", bold=True) + self._write_last_failed() + sys.exit(1) + + def run(self, suite): + result = self._run(suite) + self._exit(result.wasSuccessful()) + + +class ParallelRunner(ColouredTextRunner): + + @staticmethod + def _parallelize(suite): + def fdopen(fd, mode, *kwds): + stream = orig_fdopen(fd, mode) + atexit.register(stream.close) + return stream + + # Monkey patch concurrencytest lib bug (fdopen() stream not closed). + # https://github.com/cgoldberg/concurrencytest/issues/11 + orig_fdopen = os.fdopen + concurrencytest.os.fdopen = fdopen + forker = concurrencytest.fork_for_tests(NWORKERS) + return concurrencytest.ConcurrentTestSuite(suite, forker) + + @staticmethod + def _split_suite(suite): + serial = unittest.TestSuite() + parallel = unittest.TestSuite() + for test in suite: + if test.countTestCases() == 0: + continue + elif isinstance(test, unittest.TestSuite): + test_class = test._tests[0].__class__ + elif isinstance(test, unittest.TestCase): + test_class = test + else: + raise TypeError("can't recognize type %r" % test) + + if getattr(test_class, '_serialrun', False): + serial.addTest(test) + else: + parallel.addTest(test) + return (serial, parallel) + + def run(self, suite): + ser_suite, par_suite = self._split_suite(suite) + par_suite = self._parallelize(par_suite) + + # run parallel + cprint("starting parallel tests using %s workers" % NWORKERS, + "green", bold=True) + t = time.time() + par = self._run(par_suite) + par_elapsed = time.time() - t + + # At this point we should have N zombies (the workers), which + # will disappear with wait(). + orphans = psutil.Process().children() + gone, alive = psutil.wait_procs(orphans, timeout=1) + if alive: + cprint("alive processes %s" % alive, "red") + reap_children() + + # run serial + t = time.time() + ser = self._run(ser_suite) + ser_elapsed = time.time() - t + + # print + if not par.wasSuccessful() and ser_suite.countTestCases() > 0: + par.printErrors() # print them again at the bottom + par_fails, par_errs, par_skips = map(len, (par.failures, + par.errors, + par.skipped)) + ser_fails, ser_errs, ser_skips = map(len, (ser.failures, + ser.errors, + ser.skipped)) + print(textwrap.dedent(""" + +----------+----------+----------+----------+----------+----------+ + | | total | failures | errors | skipped | time | + +----------+----------+----------+----------+----------+----------+ + | parallel | %3s | %3s | %3s | %3s | %.2fs | + +----------+----------+----------+----------+----------+----------+ + | serial | %3s | %3s | %3s | %3s | %.2fs | + +----------+----------+----------+----------+----------+----------+ + """ % (par.testsRun, par_fails, par_errs, par_skips, par_elapsed, + ser.testsRun, ser_fails, ser_errs, ser_skips, ser_elapsed))) + print("Ran %s tests in %.3fs using %s workers" % ( + par.testsRun + ser.testsRun, par_elapsed + ser_elapsed, NWORKERS)) + ok = par.wasSuccessful() and ser.wasSuccessful() + self._exit(ok) + + +def get_runner(parallel=False): + def warn(msg): + cprint(msg + " Running serial tests instead.", "red") + if parallel: + if psutil.WINDOWS: + warn("Can't run parallel tests on Windows.") + elif concurrencytest is None: + warn("concurrencytest module is not installed.") + elif NWORKERS == 1: + warn("Only 1 CPU available.") + else: + return ParallelRunner(verbosity=VERBOSITY) + return ColouredTextRunner(verbosity=VERBOSITY) + + +# Used by test_*,py modules. +def run_from_name(name): + suite = TestLoader().from_name(name) + runner = get_runner() + runner.run(suite) + + +def setup(): + psutil._set_debug(True) + + +def main(): + setup() + usage = "python3 -m psutil.tests [opts] [test-name]" + parser = optparse.OptionParser(usage=usage, description="run unit tests") + parser.add_option("--last-failed", + action="store_true", default=False, + help="only run last failed tests") + parser.add_option("--parallel", + action="store_true", default=False, + help="run tests in parallel") + opts, args = parser.parse_args() + + if not opts.last_failed: + safe_rmpath(FAILED_TESTS_FNAME) + + # loader + loader = TestLoader() + if args: + if len(args) > 1: + parser.print_usage() + return sys.exit(1) + else: + suite = loader.from_name(args[0]) + elif opts.last_failed: + suite = loader.last_failed() else: - name = os.path.splitext(os.path.basename(name))[0] - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(name)) - return suite + suite = loader.all() + if CI_TESTING: + print_sysinfo() + runner = get_runner(opts.parallel) + runner.run(suite) -def get_suite_from_failed(): - # ...from previously failed test run - suite = unittest.TestSuite() - if not os.path.isfile(FAILED_TESTS_FNAME): - return suite - with open(FAILED_TESTS_FNAME, 'rt') as f: - names = f.read().split() - for n in names: - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(n)) - return suite - - -def save_failed_tests(result): - if result.wasSuccessful(): - return safe_rmpath(FAILED_TESTS_FNAME) - with open(FAILED_TESTS_FNAME, 'wt') as f: - for t in result.errors + result.failures: - tname = str(t[0]) - unittest.defaultTestLoader.loadTestsFromName(tname) - f.write(tname + '\n') - - -def run(name=None, last_failed=False): - setup_tests() - runner = ColouredRunner(verbosity=VERBOSITY) - suite = get_suite_from_failed() if last_failed else get_suite(name) - try: - result = runner.run(suite) - except (KeyboardInterrupt, SystemExit) as err: - print("received %s" % err.__class__.__name__, file=sys.stderr) - runner.result.printErrors() - sys.exit(1) - else: - save_failed_tests(result) - success = result.wasSuccessful() - sys.exit(0 if success else 1) + +if __name__ == '__main__': + main() diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py index 1757e3e5..a32c3f6a 100755 --- a/psutil/tests/test_aix.py +++ b/psutil/tests/test_aix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola' # Copyright (c) 2017, Arnon Yaari @@ -11,13 +11,14 @@ import re from psutil import AIX +from psutil.tests import PsutilTestCase from psutil.tests import sh from psutil.tests import unittest import psutil @unittest.skipIf(not AIX, "AIX only") -class AIXSpecificTestCase(unittest.TestCase): +class AIXSpecificTestCase(PsutilTestCase): def test_virtual_memory(self): out = sh('/usr/bin/svmon -O unit=KB') @@ -37,17 +38,17 @@ class AIXSpecificTestCase(unittest.TestCase): psutil_result = psutil.virtual_memory() - # MEMORY_TOLERANCE from psutil.tests is not enough. For some reason + # TOLERANCE_SYS_MEM from psutil.tests is not enough. For some reason # we're seeing differences of ~1.2 MB. 2 MB is still a good tolerance # when compared to GBs. - MEMORY_TOLERANCE = 2 * KB * KB # 2 MB + TOLERANCE_SYS_MEM = 2 * KB * KB # 2 MB self.assertEqual(psutil_result.total, total) self.assertAlmostEqual( - psutil_result.used, used, delta=MEMORY_TOLERANCE) + psutil_result.used, used, delta=TOLERANCE_SYS_MEM) self.assertAlmostEqual( - psutil_result.available, available, delta=MEMORY_TOLERANCE) + psutil_result.available, available, delta=TOLERANCE_SYS_MEM) self.assertAlmostEqual( - psutil_result.free, free, delta=MEMORY_TOLERANCE) + psutil_result.free, free, delta=TOLERANCE_SYS_MEM) def test_swap_memory(self): out = sh('/usr/sbin/lsps -a') @@ -117,5 +118,5 @@ class AIXSpecificTestCase(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 69fa9eee..409cc718 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -20,23 +20,25 @@ from psutil import BSD from psutil import FREEBSD from psutil import NETBSD from psutil import OPENBSD -from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY -from psutil.tests import MEMORY_TOLERANCE -from psutil.tests import reap_children +from psutil.tests import PsutilTestCase from psutil.tests import retry_on_failure from psutil.tests import sh +from psutil.tests import spawn_testproc +from psutil.tests import terminate +from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import unittest from psutil.tests import which if BSD: - PAGESIZE = os.sysconf("SC_PAGE_SIZE") - if os.getuid() == 0: # muse requires root privileges - MUSE_AVAILABLE = which('muse') - else: - MUSE_AVAILABLE = False + from psutil._psutil_posix import getpagesize + + PAGESIZE = getpagesize() + # muse requires root privileges + MUSE_AVAILABLE = True if os.getuid() == 0 and which('muse') else False else: + PAGESIZE = None MUSE_AVAILABLE = False @@ -72,16 +74,16 @@ def muse(field): @unittest.skipIf(not BSD, "BSD only") -class BSDTestCase(unittest.TestCase): +class BSDTestCase(PsutilTestCase): """Generic tests common to all BSD variants.""" @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess().pid + cls.pid = spawn_testproc().pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) @unittest.skipIf(NETBSD, "-o lstart doesn't work on NETBSD") def test_process_create_time(self): @@ -148,15 +150,15 @@ class BSDTestCase(unittest.TestCase): @unittest.skipIf(not FREEBSD, "FREEBSD only") -class FreeBSDProcessTestCase(unittest.TestCase): +class FreeBSDPsutilTestCase(PsutilTestCase): @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess().pid + cls.pid = spawn_testproc().pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) @retry_on_failure() def test_memory_maps(self): @@ -238,7 +240,7 @@ class FreeBSDProcessTestCase(unittest.TestCase): @unittest.skipIf(not FREEBSD, "FREEBSD only") -class FreeBSDSystemTestCase(unittest.TestCase): +class FreeBSDSystemTestCase(PsutilTestCase): @unittest.skipIf(not which('swapinfo'), "swapinfo util not available") def parse_swapinfo(self): @@ -279,37 +281,37 @@ class FreeBSDSystemTestCase(unittest.TestCase): def test_vmem_active(self): syst = sysctl("vm.stats.vm.v_active_count") * PAGESIZE self.assertAlmostEqual(psutil.virtual_memory().active, syst, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_inactive(self): syst = sysctl("vm.stats.vm.v_inactive_count") * PAGESIZE self.assertAlmostEqual(psutil.virtual_memory().inactive, syst, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_wired(self): syst = sysctl("vm.stats.vm.v_wire_count") * PAGESIZE self.assertAlmostEqual(psutil.virtual_memory().wired, syst, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_cached(self): syst = sysctl("vm.stats.vm.v_cache_count") * PAGESIZE self.assertAlmostEqual(psutil.virtual_memory().cached, syst, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_free(self): syst = sysctl("vm.stats.vm.v_free_count") * PAGESIZE self.assertAlmostEqual(psutil.virtual_memory().free, syst, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_buffers(self): syst = sysctl("vfs.bufspace") self.assertAlmostEqual(psutil.virtual_memory().buffers, syst, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) # --- virtual_memory(); tests against muse @@ -323,42 +325,42 @@ class FreeBSDSystemTestCase(unittest.TestCase): def test_muse_vmem_active(self): num = muse('Active') self.assertAlmostEqual(psutil.virtual_memory().active, num, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_on_failure() def test_muse_vmem_inactive(self): num = muse('Inactive') self.assertAlmostEqual(psutil.virtual_memory().inactive, num, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_on_failure() def test_muse_vmem_wired(self): num = muse('Wired') self.assertAlmostEqual(psutil.virtual_memory().wired, num, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_on_failure() def test_muse_vmem_cached(self): num = muse('Cache') self.assertAlmostEqual(psutil.virtual_memory().cached, num, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_on_failure() def test_muse_vmem_free(self): num = muse('Free') self.assertAlmostEqual(psutil.virtual_memory().free, num, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_on_failure() def test_muse_vmem_buffers(self): num = muse('Buffer') self.assertAlmostEqual(psutil.virtual_memory().buffers, num, - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) def test_cpu_stats_ctx_switches(self): self.assertAlmostEqual(psutil.cpu_stats().ctx_switches, @@ -387,17 +389,17 @@ class FreeBSDSystemTestCase(unittest.TestCase): def test_swapmem_free(self): total, used, free = self.parse_swapinfo() self.assertAlmostEqual( - psutil.swap_memory().free, free, delta=MEMORY_TOLERANCE) + psutil.swap_memory().free, free, delta=TOLERANCE_SYS_MEM) def test_swapmem_used(self): total, used, free = self.parse_swapinfo() self.assertAlmostEqual( - psutil.swap_memory().used, used, delta=MEMORY_TOLERANCE) + psutil.swap_memory().used, used, delta=TOLERANCE_SYS_MEM) def test_swapmem_total(self): total, used, free = self.parse_swapinfo() self.assertAlmostEqual( - psutil.swap_memory().total, total, delta=MEMORY_TOLERANCE) + psutil.swap_memory().total, total, delta=TOLERANCE_SYS_MEM) # --- others @@ -480,10 +482,10 @@ class FreeBSDSystemTestCase(unittest.TestCase): total, used, free = self.parse_swapinfo() self.assertAlmostEqual( sum([x.total for x in psutil.disk_swaps()]), - total, delta=MEMORY_TOLERANCE) + total, delta=TOLERANCE_SYS_MEM) self.assertAlmostEqual( sum([x.used for x in psutil.disk_swaps()]), - used, delta=MEMORY_TOLERANCE) + used, delta=TOLERANCE_SYS_MEM) lines = sh("swapinfo -k").splitlines() lines.pop(0) # header @@ -500,7 +502,7 @@ class FreeBSDSystemTestCase(unittest.TestCase): @unittest.skipIf(not OPENBSD, "OPENBSD only") -class OpenBSDTestCase(unittest.TestCase): +class OpenBSDTestCase(PsutilTestCase): def test_boot_time(self): s = sysctl('kern.boottime') @@ -515,7 +517,7 @@ class OpenBSDTestCase(unittest.TestCase): @unittest.skipIf(not NETBSD, "NETBSD only") -class NetBSDTestCase(unittest.TestCase): +class NetBSDTestCase(PsutilTestCase): @staticmethod def parse_meminfo(look_for): @@ -532,27 +534,27 @@ class NetBSDTestCase(unittest.TestCase): def test_vmem_free(self): self.assertAlmostEqual( psutil.virtual_memory().free, self.parse_meminfo("MemFree:"), - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) def test_vmem_buffers(self): self.assertAlmostEqual( psutil.virtual_memory().buffers, self.parse_meminfo("Buffers:"), - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) def test_vmem_shared(self): self.assertAlmostEqual( psutil.virtual_memory().shared, self.parse_meminfo("MemShared:"), - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) def test_swapmem_total(self): self.assertAlmostEqual( psutil.swap_memory().total, self.parse_meminfo("SwapTotal:"), - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) def test_swapmem_free(self): self.assertAlmostEqual( psutil.swap_memory().free, self.parse_meminfo("SwapFree:"), - delta=MEMORY_TOLERANCE) + delta=TOLERANCE_SYS_MEM) def test_swapmem_used(self): smem = psutil.swap_memory() @@ -582,5 +584,5 @@ class NetBSDTestCase(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index c7fe1992..6bbf2194 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -6,10 +6,9 @@ """Tests for net_connections() and Process.connections() APIs.""" -import contextlib -import errno import os import socket +import sys import textwrap from contextlib import closing from socket import AF_INET @@ -31,42 +30,36 @@ from psutil._compat import PY3 from psutil.tests import AF_UNIX from psutil.tests import bind_socket from psutil.tests import bind_unix_socket -from psutil.tests import check_net_address -from psutil.tests import CIRRUS +from psutil.tests import check_connection_ntuple from psutil.tests import create_sockets -from psutil.tests import enum -from psutil.tests import get_free_port from psutil.tests import HAS_CONNECTIONS_UNIX -from psutil.tests import pyrun +from psutil.tests import PsutilTestCase from psutil.tests import reap_children -from psutil.tests import safe_rmpath +from psutil.tests import retry_on_failure +from psutil.tests import serialrun from psutil.tests import skip_on_access_denied from psutil.tests import SKIP_SYSCONS from psutil.tests import tcp_socketpair -from psutil.tests import TESTFN -from psutil.tests import TRAVIS from psutil.tests import unittest -from psutil.tests import unix_socket_path from psutil.tests import unix_socketpair from psutil.tests import wait_for_file thisproc = psutil.Process() SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) +PYTHON_39 = sys.version_info[:2] == (3, 9) -class Base(object): +@serialrun +class ConnectionTestCase(PsutilTestCase): def setUp(self): - safe_rmpath(TESTFN) if not (NETBSD or FREEBSD): # process opens a UNIX socket to /var/log/run. cons = thisproc.connections(kind='all') assert not cons, cons def tearDown(self): - safe_rmpath(TESTFN) - reap_children() if not (FREEBSD or NETBSD): # Make sure we closed all resources. # NetBSD opens a UNIX socket to /var/log/run. @@ -93,100 +86,27 @@ class Base(object): proc_cons.sort() self.assertEqual(proc_cons, sys_cons) - def check_connection_ntuple(self, conn): - """Check validity of a connection namedtuple.""" - def check_ntuple(conn): - has_pid = len(conn) == 7 - self.assertIn(len(conn), (6, 7)) - self.assertEqual(conn[0], conn.fd) - self.assertEqual(conn[1], conn.family) - self.assertEqual(conn[2], conn.type) - self.assertEqual(conn[3], conn.laddr) - self.assertEqual(conn[4], conn.raddr) - self.assertEqual(conn[5], conn.status) - if has_pid: - self.assertEqual(conn[6], conn.pid) - - def check_family(conn): - self.assertIn(conn.family, (AF_INET, AF_INET6, AF_UNIX)) - if enum is not None: - assert isinstance(conn.family, enum.IntEnum), conn - else: - assert isinstance(conn.family, int), conn - if conn.family == AF_INET: - # actually try to bind the local socket; ignore IPv6 - # sockets as their address might be represented as - # an IPv4-mapped-address (e.g. "::127.0.0.1") - # and that's rejected by bind() - s = socket.socket(conn.family, conn.type) - with contextlib.closing(s): - try: - s.bind((conn.laddr[0], 0)) - except socket.error as err: - if err.errno != errno.EADDRNOTAVAIL: - raise - elif conn.family == AF_UNIX: - self.assertEqual(conn.status, psutil.CONN_NONE) - - def check_type(conn): - # SOCK_SEQPACKET may happen in case of AF_UNIX socks - self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET)) - if enum is not None: - assert isinstance(conn.type, enum.IntEnum), conn - else: - assert isinstance(conn.type, int), conn - if conn.type == SOCK_DGRAM: - self.assertEqual(conn.status, psutil.CONN_NONE) - - def check_addrs(conn): - # check IP address and port sanity - for addr in (conn.laddr, conn.raddr): - if conn.family in (AF_INET, AF_INET6): - self.assertIsInstance(addr, tuple) - if not addr: - continue - self.assertIsInstance(addr.port, int) - assert 0 <= addr.port <= 65535, addr.port - check_net_address(addr.ip, conn.family) - elif conn.family == AF_UNIX: - self.assertIsInstance(addr, str) - - def check_status(conn): - self.assertIsInstance(conn.status, str) - valids = [getattr(psutil, x) for x in dir(psutil) - if x.startswith('CONN_')] - self.assertIn(conn.status, valids) - if conn.family in (AF_INET, AF_INET6) and conn.type == SOCK_STREAM: - self.assertNotEqual(conn.status, psutil.CONN_NONE) - else: - self.assertEqual(conn.status, psutil.CONN_NONE) - - check_ntuple(conn) - check_family(conn) - check_type(conn) - check_addrs(conn) - check_status(conn) - -class TestBase(Base, unittest.TestCase): +class TestBasicOperations(ConnectionTestCase): @unittest.skipIf(SKIP_SYSCONS, "requires root") def test_system(self): with create_sockets(): for conn in psutil.net_connections(kind='all'): - self.check_connection_ntuple(conn) + check_connection_ntuple(conn) def test_process(self): with create_sockets(): for conn in psutil.Process().connections(kind='all'): - self.check_connection_ntuple(conn) + check_connection_ntuple(conn) def test_invalid_kind(self): self.assertRaises(ValueError, thisproc.connections, kind='???') self.assertRaises(ValueError, psutil.net_connections, kind='???') -class TestUnconnectedSockets(Base, unittest.TestCase): +@serialrun +class TestUnconnectedSockets(ConnectionTestCase): """Tests sockets which are open but not connected to anything.""" def get_conn_from_sock(self, sock): @@ -208,7 +128,7 @@ class TestUnconnectedSockets(Base, unittest.TestCase): only (the one supposed to be checked). """ conn = self.get_conn_from_sock(sock) - self.check_connection_ntuple(conn) + check_connection_ntuple(conn) # fd, family, type if conn.fd != -1: @@ -238,7 +158,7 @@ class TestUnconnectedSockets(Base, unittest.TestCase): return conn def test_tcp_v4(self): - addr = ("127.0.0.1", get_free_port()) + addr = ("127.0.0.1", 0) with closing(bind_socket(AF_INET, SOCK_STREAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr @@ -246,14 +166,14 @@ class TestUnconnectedSockets(Base, unittest.TestCase): @unittest.skipIf(not supports_ipv6(), "IPv6 not supported") def test_tcp_v6(self): - addr = ("::1", get_free_port()) + addr = ("::1", 0) with closing(bind_socket(AF_INET6, SOCK_STREAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_LISTEN) def test_udp_v4(self): - addr = ("127.0.0.1", get_free_port()) + addr = ("127.0.0.1", 0) with closing(bind_socket(AF_INET, SOCK_DGRAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr @@ -261,7 +181,7 @@ class TestUnconnectedSockets(Base, unittest.TestCase): @unittest.skipIf(not supports_ipv6(), "IPv6 not supported") def test_udp_v6(self): - addr = ("::1", get_free_port()) + addr = ("::1", 0) with closing(bind_socket(AF_INET6, SOCK_DGRAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr @@ -269,22 +189,23 @@ class TestUnconnectedSockets(Base, unittest.TestCase): @unittest.skipIf(not POSIX, 'POSIX only') def test_unix_tcp(self): - with unix_socket_path() as name: - with closing(bind_unix_socket(name, type=SOCK_STREAM)) as sock: - conn = self.check_socket(sock) - assert not conn.raddr - self.assertEqual(conn.status, psutil.CONN_NONE) + testfn = self.get_testfn() + with closing(bind_unix_socket(testfn, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_NONE) @unittest.skipIf(not POSIX, 'POSIX only') def test_unix_udp(self): - with unix_socket_path() as name: - with closing(bind_unix_socket(name, type=SOCK_STREAM)) as sock: - conn = self.check_socket(sock) - assert not conn.raddr - self.assertEqual(conn.status, psutil.CONN_NONE) + testfn = self.get_testfn() + with closing(bind_unix_socket(testfn, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_NONE) -class TestConnectedSocket(Base, unittest.TestCase): +@serialrun +class TestConnectedSocket(ConnectionTestCase): """Test socket pairs which are are actually connected to each other. """ @@ -293,7 +214,7 @@ class TestConnectedSocket(Base, unittest.TestCase): # in TIME_WAIT state. @unittest.skipIf(SUNOS, "unreliable on SUONS") def test_tcp(self): - addr = ("127.0.0.1", get_free_port()) + addr = ("127.0.0.1", 0) assert not thisproc.connections(kind='tcp4') server, client = tcp_socketpair(AF_INET, addr=addr) try: @@ -313,42 +234,39 @@ class TestConnectedSocket(Base, unittest.TestCase): @unittest.skipIf(not POSIX, 'POSIX only') def test_unix(self): - with unix_socket_path() as name: - server, client = unix_socketpair(name) - try: - cons = thisproc.connections(kind='unix') - assert not (cons[0].laddr and cons[0].raddr) - assert not (cons[1].laddr and cons[1].raddr) - if NETBSD or FREEBSD: - # On NetBSD creating a UNIX socket will cause - # a UNIX connection to /var/run/log. - cons = [c for c in cons if c.raddr != '/var/run/log'] - if CIRRUS: - cons = [c for c in cons if c.fd in - (server.fileno(), client.fileno())] - self.assertEqual(len(cons), 2, msg=cons) - if LINUX or FREEBSD or SUNOS: - # remote path is never set - self.assertEqual(cons[0].raddr, "") - self.assertEqual(cons[1].raddr, "") - # one local address should though - self.assertEqual(name, cons[0].laddr or cons[1].laddr) - elif OPENBSD: - # No addresses whatsoever here. - for addr in (cons[0].laddr, cons[0].raddr, - cons[1].laddr, cons[1].raddr): - self.assertEqual(addr, "") - else: - # On other systems either the laddr or raddr - # of both peers are set. - self.assertEqual(cons[0].laddr or cons[1].laddr, name) - self.assertEqual(cons[0].raddr or cons[1].raddr, name) - finally: - server.close() - client.close() + testfn = self.get_testfn() + server, client = unix_socketpair(testfn) + try: + cons = thisproc.connections(kind='unix') + assert not (cons[0].laddr and cons[0].raddr) + assert not (cons[1].laddr and cons[1].raddr) + if NETBSD or FREEBSD: + # On NetBSD creating a UNIX socket will cause + # a UNIX connection to /var/run/log. + cons = [c for c in cons if c.raddr != '/var/run/log'] + self.assertEqual(len(cons), 2, msg=cons) + if LINUX or FREEBSD or SUNOS: + # remote path is never set + self.assertEqual(cons[0].raddr, "") + self.assertEqual(cons[1].raddr, "") + # one local address should though + self.assertEqual(testfn, cons[0].laddr or cons[1].laddr) + elif OPENBSD: + # No addresses whatsoever here. + for addr in (cons[0].laddr, cons[0].raddr, + cons[1].laddr, cons[1].raddr): + self.assertEqual(addr, "") + else: + # On other systems either the laddr or raddr + # of both peers are set. + self.assertEqual(cons[0].laddr or cons[1].laddr, testfn) + self.assertEqual(cons[0].raddr or cons[1].raddr, testfn) + finally: + server.close() + client.close() -class TestFilters(Base, unittest.TestCase): +class TestFilters(ConnectionTestCase): def test_filters(self): def check(kind, families, types): @@ -395,10 +313,12 @@ class TestFilters(Base, unittest.TestCase): @skip_on_access_denied(only_if=MACOS) def test_combos(self): + reap_children() + def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): all_kinds = ("all", "inet", "inet4", "inet6", "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6") - self.check_connection_ntuple(conn) + check_connection_ntuple(conn) self.assertEqual(conn.family, family) self.assertEqual(conn.type, type) self.assertEqual(conn.laddr, laddr) @@ -418,45 +338,45 @@ class TestFilters(Base, unittest.TestCase): tcp_template = textwrap.dedent(""" import socket, time - s = socket.socket($family, socket.SOCK_STREAM) - s.bind(('$addr', 0)) + s = socket.socket({family}, socket.SOCK_STREAM) + s.bind(('{addr}', 0)) s.listen(5) - with open('$testfn', 'w') as f: + with open('{testfn}', 'w') as f: f.write(str(s.getsockname()[:2])) time.sleep(60) - """) + """) udp_template = textwrap.dedent(""" import socket, time - s = socket.socket($family, socket.SOCK_DGRAM) - s.bind(('$addr', 0)) - with open('$testfn', 'w') as f: + s = socket.socket({family}, socket.SOCK_DGRAM) + s.bind(('{addr}', 0)) + with open('{testfn}', 'w') as f: f.write(str(s.getsockname()[:2])) time.sleep(60) - """) + """) - from string import Template - testfile = os.path.basename(TESTFN) - tcp4_template = Template(tcp_template).substitute( + # must be relative on Windows + testfile = os.path.basename(self.get_testfn(dir=os.getcwd())) + tcp4_template = tcp_template.format( family=int(AF_INET), addr="127.0.0.1", testfn=testfile) - udp4_template = Template(udp_template).substitute( + udp4_template = udp_template.format( family=int(AF_INET), addr="127.0.0.1", testfn=testfile) - tcp6_template = Template(tcp_template).substitute( + tcp6_template = tcp_template.format( family=int(AF_INET6), addr="::1", testfn=testfile) - udp6_template = Template(udp_template).substitute( + udp6_template = udp_template.format( family=int(AF_INET6), addr="::1", testfn=testfile) # launch various subprocess instantiating a socket of various # families and types to enrich psutil results - tcp4_proc = pyrun(tcp4_template) - tcp4_addr = eval(wait_for_file(testfile)) - udp4_proc = pyrun(udp4_template) - udp4_addr = eval(wait_for_file(testfile)) + tcp4_proc = self.pyrun(tcp4_template) + tcp4_addr = eval(wait_for_file(testfile, delete=True)) + udp4_proc = self.pyrun(udp4_template) + udp4_addr = eval(wait_for_file(testfile, delete=True)) if supports_ipv6(): - tcp6_proc = pyrun(tcp6_template) - tcp6_addr = eval(wait_for_file(testfile)) - udp6_proc = pyrun(udp6_template) - udp6_addr = eval(wait_for_file(testfile)) + tcp6_proc = self.pyrun(tcp6_template) + tcp6_addr = eval(wait_for_file(testfile, delete=True)) + udp6_proc = self.pyrun(udp6_template) + udp6_addr = eval(wait_for_file(testfile, delete=True)) else: tcp6_proc = None udp6_proc = None @@ -548,7 +468,7 @@ class TestFilters(Base, unittest.TestCase): @unittest.skipIf(SKIP_SYSCONS, "requires root") -class TestSystemWideConnections(Base, unittest.TestCase): +class TestSystemWideConnections(ConnectionTestCase): """Tests for net_connections().""" def test_it(self): @@ -557,7 +477,7 @@ class TestSystemWideConnections(Base, unittest.TestCase): self.assertIn(conn.family, families, msg=conn) if conn.family != AF_UNIX: self.assertIn(conn.type, types_, msg=conn) - self.check_connection_ntuple(conn) + check_connection_ntuple(conn) with create_sockets(): from psutil._common import conn_tmap @@ -570,8 +490,7 @@ class TestSystemWideConnections(Base, unittest.TestCase): self.assertEqual(len(cons), len(set(cons))) check(cons, families, types_) - # See: https://travis-ci.org/giampaolo/psutil/jobs/237566297 - @unittest.skipIf(MACOS and TRAVIS, "unreliable on MACOS + TRAVIS") + @retry_on_failure() def test_multi_sockets_procs(self): # Creates multiple sub processes, each creating different # sockets. For each process check that proc.connections() @@ -583,23 +502,23 @@ class TestSystemWideConnections(Base, unittest.TestCase): expected = len(socks) pids = [] times = 10 + fnames = [] for i in range(times): - fname = os.path.realpath(TESTFN) + str(i) + fname = self.get_testfn() + fnames.append(fname) src = textwrap.dedent("""\ import time, os from psutil.tests import create_sockets with create_sockets(): with open(r'%s', 'w') as f: - f.write(str(os.getpid())) + f.write("hello") time.sleep(60) """ % fname) - sproc = pyrun(src) + sproc = self.pyrun(src) pids.append(sproc.pid) - self.addCleanup(safe_rmpath, fname) # sync - for i in range(times): - fname = TESTFN + str(i) + for fname in fnames: wait_for_file(fname) syscons = [x for x in psutil.net_connections(kind='all') if x.pid @@ -611,7 +530,7 @@ class TestSystemWideConnections(Base, unittest.TestCase): self.assertEqual(len(p.connections('all')), expected) -class TestMisc(unittest.TestCase): +class TestMisc(PsutilTestCase): def test_connection_constants(self): ints = [] @@ -633,5 +552,5 @@ class TestMisc(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 85f9c00d..51845662 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -10,11 +10,13 @@ Some of these are duplicates of tests test_system.py and test_process.py """ import errno +import multiprocessing import os +import signal import stat +import sys import time import traceback -import warnings from psutil import AIX from psutil import BSD @@ -27,22 +29,28 @@ from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS +from psutil._compat import FileNotFoundError from psutil._compat import long +from psutil._compat import range +from psutil.tests import APPVEYOR +from psutil.tests import check_connection_ntuple +from psutil.tests import CI_TESTING from psutil.tests import create_sockets from psutil.tests import enum -from psutil.tests import get_kernel_version +from psutil.tests import GITHUB_ACTIONS from psutil.tests import HAS_CPU_FREQ from psutil.tests import HAS_NET_IO_COUNTERS -from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import is_namedtuple -from psutil.tests import safe_rmpath +from psutil.tests import kernel_version +from psutil.tests import process_namespace +from psutil.tests import PsutilTestCase +from psutil.tests import PYPY +from psutil.tests import serialrun from psutil.tests import SKIP_SYSCONS -from psutil.tests import TESTFN from psutil.tests import unittest from psutil.tests import VALID_PROC_STATUSES -from psutil.tests import warn import psutil @@ -53,7 +61,7 @@ import psutil # Make sure code reflects what doc promises in terms of APIs # availability. -class TestAvailConstantsAPIs(unittest.TestCase): +class TestAvailConstantsAPIs(PsutilTestCase): def test_PROCFS_PATH(self): self.assertEqual(hasattr(psutil, "PROCFS_PATH"), @@ -82,32 +90,41 @@ class TestAvailConstantsAPIs(unittest.TestCase): ae(hasattr(psutil, "IOPRIO_LOW"), WINDOWS) ae(hasattr(psutil, "IOPRIO_VERYLOW"), WINDOWS) - def test_linux_rlimit(self): + @unittest.skipIf(GITHUB_ACTIONS and LINUX, + "unsupported on GITHUB_ACTIONS + LINUX") + def test_rlimit(self): ae = self.assertEqual - hasit = LINUX and get_kernel_version() >= (2, 6, 36) - ae(hasattr(psutil.Process, "rlimit"), hasit) - ae(hasattr(psutil, "RLIM_INFINITY"), hasit) - ae(hasattr(psutil, "RLIMIT_AS"), hasit) - ae(hasattr(psutil, "RLIMIT_CORE"), hasit) - ae(hasattr(psutil, "RLIMIT_CPU"), hasit) - ae(hasattr(psutil, "RLIMIT_DATA"), hasit) - ae(hasattr(psutil, "RLIMIT_FSIZE"), hasit) - ae(hasattr(psutil, "RLIMIT_LOCKS"), hasit) - ae(hasattr(psutil, "RLIMIT_MEMLOCK"), hasit) - ae(hasattr(psutil, "RLIMIT_NOFILE"), hasit) - ae(hasattr(psutil, "RLIMIT_NPROC"), hasit) - ae(hasattr(psutil, "RLIMIT_RSS"), hasit) - ae(hasattr(psutil, "RLIMIT_STACK"), hasit) - - hasit = LINUX and get_kernel_version() >= (3, 0) - ae(hasattr(psutil, "RLIMIT_MSGQUEUE"), hasit) - ae(hasattr(psutil, "RLIMIT_NICE"), hasit) - ae(hasattr(psutil, "RLIMIT_RTPRIO"), hasit) - ae(hasattr(psutil, "RLIMIT_RTTIME"), hasit) - ae(hasattr(psutil, "RLIMIT_SIGPENDING"), hasit) - - -class TestAvailSystemAPIs(unittest.TestCase): + ae(hasattr(psutil, "RLIM_INFINITY"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_AS"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_CORE"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_CPU"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_DATA"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_FSIZE"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_MEMLOCK"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_NOFILE"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_NPROC"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_RSS"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_STACK"), LINUX or FREEBSD) + + ae(hasattr(psutil, "RLIMIT_LOCKS"), LINUX) + if POSIX: + if kernel_version() >= (2, 6, 8): + ae(hasattr(psutil, "RLIMIT_MSGQUEUE"), LINUX) + if kernel_version() >= (2, 6, 12): + ae(hasattr(psutil, "RLIMIT_NICE"), LINUX) + if kernel_version() >= (2, 6, 12): + ae(hasattr(psutil, "RLIMIT_RTPRIO"), LINUX) + if kernel_version() >= (2, 6, 25): + ae(hasattr(psutil, "RLIMIT_RTTIME"), LINUX) + if kernel_version() >= (2, 6, 8): + ae(hasattr(psutil, "RLIMIT_SIGPENDING"), LINUX) + + ae(hasattr(psutil, "RLIMIT_SWAP"), FREEBSD) + ae(hasattr(psutil, "RLIMIT_SBSIZE"), FREEBSD) + ae(hasattr(psutil, "RLIMIT_NPTS"), FREEBSD) + + +class TestAvailSystemAPIs(PsutilTestCase): def test_win_service_iter(self): self.assertEqual(hasattr(psutil, "win_service_iter"), WINDOWS) @@ -116,11 +133,8 @@ class TestAvailSystemAPIs(unittest.TestCase): self.assertEqual(hasattr(psutil, "win_service_get"), WINDOWS) def test_cpu_freq(self): - linux = (LINUX and - (os.path.exists("/sys/devices/system/cpu/cpufreq") or - os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"))) self.assertEqual(hasattr(psutil, "cpu_freq"), - linux or MACOS or WINDOWS or FREEBSD) + LINUX or MACOS or WINDOWS or FREEBSD) def test_sensors_temperatures(self): self.assertEqual( @@ -138,11 +152,12 @@ class TestAvailSystemAPIs(unittest.TestCase): self.assertEqual(hasattr(psutil, "disk_swaps"), hasit) -class TestAvailProcessAPIs(unittest.TestCase): +class TestAvailProcessAPIs(PsutilTestCase): def test_environ(self): self.assertEqual(hasattr(psutil.Process, "environ"), - LINUX or MACOS or WINDOWS or AIX or SUNOS) + LINUX or MACOS or WINDOWS or AIX or SUNOS or + FREEBSD or OPENBSD or NETBSD) def test_uids(self): self.assertEqual(hasattr(psutil.Process, "uids"), POSIX) @@ -156,8 +171,10 @@ class TestAvailProcessAPIs(unittest.TestCase): def test_ionice(self): self.assertEqual(hasattr(psutil.Process, "ionice"), LINUX or WINDOWS) + @unittest.skipIf(GITHUB_ACTIONS and LINUX, + "unsupported on GITHUB_ACTIONS + LINUX") def test_rlimit(self): - self.assertEqual(hasattr(psutil.Process, "rlimit"), LINUX) + self.assertEqual(hasattr(psutil.Process, "rlimit"), LINUX or FREEBSD) def test_io_counters(self): hasit = hasattr(psutil.Process, "io_counters") @@ -184,27 +201,11 @@ class TestAvailProcessAPIs(unittest.TestCase): # =================================================================== -# --- Test deprecations +# --- API types # =================================================================== -class TestDeprecations(unittest.TestCase): - - def test_memory_info_ex(self): - with warnings.catch_warnings(record=True) as ws: - psutil.Process().memory_info_ex() - w = ws[0] - self.assertIsInstance(w.category(), DeprecationWarning) - self.assertIn("memory_info_ex() is deprecated", str(w.message)) - self.assertIn("use memory_info() instead", str(w.message)) - - -# =================================================================== -# --- System API types -# =================================================================== - - -class TestSystemAPITypes(unittest.TestCase): +class TestSystemAPITypes(PsutilTestCase): """Check the return types of system related APIs. Mainly we want to test we never return unicode on Python 2, see: https://github.com/giampaolo/psutil/issues/1039 @@ -214,9 +215,6 @@ class TestSystemAPITypes(unittest.TestCase): def setUpClass(cls): cls.proc = psutil.Process() - def tearDown(self): - safe_rmpath(TESTFN) - def assert_ntuple_of_nums(self, nt, type_=float, gezero=True): assert is_namedtuple(nt) for n in nt: @@ -259,6 +257,8 @@ class TestSystemAPITypes(unittest.TestCase): self.assertIsInstance(disk.mountpoint, str) self.assertIsInstance(disk.fstype, str) self.assertIsInstance(disk.opts, str) + self.assertIsInstance(disk.maxfile, int) + self.assertIsInstance(disk.maxpath, int) @unittest.skipIf(SKIP_SYSCONS, "requires root") def test_net_connections(self): @@ -273,10 +273,10 @@ class TestSystemAPITypes(unittest.TestCase): for ifname, addrs in psutil.net_if_addrs().items(): self.assertIsInstance(ifname, str) for addr in addrs: - if enum is not None: - assert isinstance(addr.family, enum.IntEnum), addr + if enum is not None and not PYPY: + self.assertIsInstance(addr.family, enum.IntEnum) else: - assert isinstance(addr.family, int), addr + self.assertIsInstance(addr.family, int) self.assertIsInstance(addr.address, str) self.assertIsInstance(addr.netmask, (str, type(None))) self.assertIsInstance(addr.broadcast, (str, type(None))) @@ -332,105 +332,115 @@ class TestSystemAPITypes(unittest.TestCase): self.assertIsInstance(user.pid, (int, type(None))) +class TestProcessWaitType(PsutilTestCase): + + @unittest.skipIf(not POSIX, "not POSIX") + def test_negative_signal(self): + p = psutil.Process(self.spawn_testproc().pid) + p.terminate() + code = p.wait() + self.assertEqual(code, -signal.SIGTERM) + if enum is not None: + self.assertIsInstance(code, enum.IntEnum) + else: + self.assertIsInstance(code, int) + + # =================================================================== # --- Featch all processes test # =================================================================== -class TestFetchAllProcesses(unittest.TestCase): +def proc_info(pid): + tcase = PsutilTestCase() + + def check_exception(exc, proc, name, ppid): + tcase.assertEqual(exc.pid, pid) + tcase.assertEqual(exc.name, name) + if isinstance(exc, psutil.ZombieProcess): + if exc.ppid is not None: + tcase.assertGreaterEqual(exc.ppid, 0) + tcase.assertEqual(exc.ppid, ppid) + elif isinstance(exc, psutil.NoSuchProcess): + tcase.assertProcessGone(proc) + str(exc) + + def do_wait(): + if pid != 0: + try: + proc.wait(0) + except psutil.Error as exc: + check_exception(exc, proc, name, ppid) + + try: + proc = psutil.Process(pid) + d = proc.as_dict(['ppid', 'name']) + except psutil.NoSuchProcess: + return {} + + name, ppid = d['name'], d['ppid'] + info = {'pid': proc.pid} + ns = process_namespace(proc) + # We don't use oneshot() because in order not to fool + # check_exception() in case of NSP. + for fun, fun_name in ns.iter(ns.getters, clear_cache=False): + try: + info[fun_name] = fun() + except psutil.Error as exc: + check_exception(exc, proc, name, ppid) + continue + do_wait() + return info + + +@serialrun +class TestFetchAllProcesses(PsutilTestCase): """Test which iterates over all running processes and performs some sanity checks against Process API's returned values. + Uses a process pool to get info about all processes. """ - def get_attr_names(self): - excluded_names = set([ - 'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', - 'as_dict', 'parent', 'parents', 'children', 'memory_info_ex', - 'oneshot', - ]) - if LINUX and not HAS_RLIMIT: - excluded_names.add('rlimit') - attrs = [] - for name in dir(psutil.Process): - if name.startswith("_"): - continue - if name in excluded_names: - continue - attrs.append(name) - return attrs - - def iter_procs(self): - attrs = self.get_attr_names() - for p in psutil.process_iter(): - with p.oneshot(): - for name in attrs: - yield (p, name) - - def call_meth(self, p, name): - args = () - kwargs = {} - attr = getattr(p, name, None) - if attr is not None and callable(attr): - if name == 'rlimit': - args = (psutil.RLIMIT_NOFILE,) - elif name == 'memory_maps': - kwargs = {'grouped': False} - return attr(*args, **kwargs) - else: - return attr + def setUp(self): + self.pool = multiprocessing.Pool() - def test_fetch_all(self): - valid_procs = 0 - default = object() + def tearDown(self): + self.pool.terminate() + self.pool.join() + + def iter_proc_info(self): + # Fixes "can't pickle <function proc_info>: it's not the + # same object as test_contracts.proc_info". + from psutil.tests.test_contracts import proc_info + return self.pool.imap_unordered(proc_info, psutil.pids()) + + def test_all(self): failures = [] - for p, name in self.iter_procs(): - ret = default - try: - ret = self.call_meth(p, name) - except NotImplementedError: - msg = "%r was skipped because not implemented" % ( - self.__class__.__name__ + '.test_' + name) - warn(msg) - except (psutil.NoSuchProcess, psutil.AccessDenied) as err: - self.assertEqual(err.pid, p.pid) - if err.name: - # make sure exception's name attr is set - # with the actual process name - self.assertEqual(err.name, p.name()) - assert str(err) - assert err.msg - except Exception: - s = '\n' + '=' * 70 + '\n' - s += "FAIL: test_%s (proc=%s" % (name, p) - if ret != default: - s += ", ret=%s)" % repr(ret) - s += ')\n' - s += '-' * 70 - s += "\n%s" % traceback.format_exc() - s = "\n".join((" " * 4) + i for i in s.splitlines()) - s += '\n' - failures.append(s) - break - else: - valid_procs += 1 - if ret not in (0, 0.0, [], None, '', {}): - assert ret, ret + for info in self.iter_proc_info(): + for name, value in info.items(): meth = getattr(self, name) - meth(ret, p) - + try: + meth(value, info) + except AssertionError: + s = '\n' + '=' * 70 + '\n' + s += "FAIL: test_%s pid=%s, ret=%s\n" % ( + name, info['pid'], repr(value)) + s += '-' * 70 + s += "\n%s" % traceback.format_exc() + s = "\n".join((" " * 4) + i for i in s.splitlines()) + s += '\n' + failures.append(s) + else: + if value not in (0, 0.0, [], None, '', {}): + assert value, value if failures: - self.fail(''.join(failures)) - - # we should always have a non-empty list, not including PID 0 etc. - # special cases. - assert valid_procs + raise self.fail(''.join(failures)) - def cmdline(self, ret, proc): + def cmdline(self, ret, info): self.assertIsInstance(ret, list) for part in ret: self.assertIsInstance(part, str) - def exe(self, ret, proc): + def exe(self, ret, info): self.assertIsInstance(ret, (str, type(None))) if not ret: self.assertEqual(ret, '') @@ -443,30 +453,36 @@ class TestFetchAllProcesses(unittest.TestCase): # http://stackoverflow.com/questions/3112546/os-path-exists-lies if POSIX and os.path.isfile(ret): if hasattr(os, 'access') and hasattr(os, "X_OK"): - # XXX may fail on MACOS - assert os.access(ret, os.X_OK) - - def pid(self, ret, proc): + # XXX: may fail on MACOS + try: + assert os.access(ret, os.X_OK) + except AssertionError: + if os.path.exists(ret) and not CI_TESTING: + raise + + def pid(self, ret, info): self.assertIsInstance(ret, int) self.assertGreaterEqual(ret, 0) - def ppid(self, ret, proc): + def ppid(self, ret, info): self.assertIsInstance(ret, (int, long)) self.assertGreaterEqual(ret, 0) - def name(self, ret, proc): + def name(self, ret, info): self.assertIsInstance(ret, str) + if APPVEYOR and not ret and info['status'] == 'stopped': + return # on AIX, "<exiting>" processes don't have names if not AIX: assert ret - def create_time(self, ret, proc): + def create_time(self, ret, info): self.assertIsInstance(ret, float) try: self.assertGreaterEqual(ret, 0) except AssertionError: # XXX - if OPENBSD and proc.status() == psutil.STATUS_ZOMBIE: + if OPENBSD and info['status'] == psutil.STATUS_ZOMBIE: pass else: raise @@ -476,13 +492,13 @@ class TestFetchAllProcesses(unittest.TestCase): # with strftime time.strftime("%Y %m %d %H:%M:%S", time.localtime(ret)) - def uids(self, ret, proc): + def uids(self, ret, info): assert is_namedtuple(ret) for uid in ret: self.assertIsInstance(uid, int) self.assertGreaterEqual(uid, 0) - def gids(self, ret, proc): + def gids(self, ret, info): assert is_namedtuple(ret) # note: testing all gids as above seems not to be reliable for # gid == 30 (nodoby); not sure why. @@ -491,24 +507,24 @@ class TestFetchAllProcesses(unittest.TestCase): if not MACOS and not NETBSD: self.assertGreaterEqual(gid, 0) - def username(self, ret, proc): + def username(self, ret, info): self.assertIsInstance(ret, str) assert ret - def status(self, ret, proc): + def status(self, ret, info): self.assertIsInstance(ret, str) assert ret self.assertNotEqual(ret, '?') # XXX self.assertIn(ret, VALID_PROC_STATUSES) - def io_counters(self, ret, proc): + def io_counters(self, ret, info): assert is_namedtuple(ret) for field in ret: self.assertIsInstance(field, (int, long)) if field != -1: self.assertGreaterEqual(field, 0) - def ionice(self, ret, proc): + def ionice(self, ret, info): if LINUX: self.assertIsInstance(ret.ioclass, int) self.assertIsInstance(ret.value, int) @@ -524,11 +540,13 @@ class TestFetchAllProcesses(unittest.TestCase): self.assertGreaterEqual(ret, 0) self.assertIn(ret, choices) - def num_threads(self, ret, proc): + def num_threads(self, ret, info): self.assertIsInstance(ret, int) + if APPVEYOR and not ret and info['status'] == 'stopped': + return self.assertGreaterEqual(ret, 1) - def threads(self, ret, proc): + def threads(self, ret, info): self.assertIsInstance(ret, list) for t in ret: assert is_namedtuple(t) @@ -538,18 +556,18 @@ class TestFetchAllProcesses(unittest.TestCase): for field in t: self.assertIsInstance(field, (int, float)) - def cpu_times(self, ret, proc): + def cpu_times(self, ret, info): assert is_namedtuple(ret) for n in ret: self.assertIsInstance(n, float) self.assertGreaterEqual(n, 0) # TODO: check ntuple fields - def cpu_percent(self, ret, proc): + def cpu_percent(self, ret, info): self.assertIsInstance(ret, float) assert 0.0 <= ret <= 100.0, ret - def cpu_num(self, ret, proc): + def cpu_num(self, ret, info): self.assertIsInstance(ret, int) if FREEBSD and ret == -1: return @@ -558,7 +576,7 @@ class TestFetchAllProcesses(unittest.TestCase): self.assertEqual(ret, 0) self.assertIn(ret, list(range(psutil.cpu_count()))) - def memory_info(self, ret, proc): + def memory_info(self, ret, info): assert is_namedtuple(ret) for value in ret: self.assertIsInstance(value, (int, long)) @@ -569,7 +587,7 @@ class TestFetchAllProcesses(unittest.TestCase): self.assertGreaterEqual(ret.peak_nonpaged_pool, ret.nonpaged_pool) self.assertGreaterEqual(ret.peak_pagefile, ret.pagefile) - def memory_full_info(self, ret, proc): + def memory_full_info(self, ret, info): assert is_namedtuple(ret) total = psutil.virtual_memory().total for name in ret._fields: @@ -585,7 +603,7 @@ class TestFetchAllProcesses(unittest.TestCase): if LINUX: self.assertGreaterEqual(ret.pss, ret.uss) - def open_files(self, ret, proc): + def open_files(self, ret, info): self.assertIsInstance(ret, list) for f in ret: self.assertIsInstance(f.fd, int) @@ -603,19 +621,25 @@ class TestFetchAllProcesses(unittest.TestCase): # XXX see: https://github.com/giampaolo/psutil/issues/595 continue assert os.path.isabs(f.path), f - assert os.path.isfile(f.path), f + try: + st = os.stat(f.path) + except FileNotFoundError: + pass + else: + assert stat.S_ISREG(st.st_mode), f - def num_fds(self, ret, proc): + def num_fds(self, ret, info): self.assertIsInstance(ret, int) self.assertGreaterEqual(ret, 0) - def connections(self, ret, proc): + def connections(self, ret, info): with create_sockets(): self.assertEqual(len(ret), len(set(ret))) for conn in ret: assert is_namedtuple(conn) + check_connection_ntuple(conn) - def cwd(self, ret, proc): + def cwd(self, ret, info): if ret: # 'ret' can be None or empty self.assertIsInstance(ret, str) assert os.path.isabs(ret), ret @@ -631,28 +655,28 @@ class TestFetchAllProcesses(unittest.TestCase): else: assert stat.S_ISDIR(st.st_mode) - def memory_percent(self, ret, proc): + def memory_percent(self, ret, info): self.assertIsInstance(ret, float) assert 0 <= ret <= 100, ret - def is_running(self, ret, proc): + def is_running(self, ret, info): self.assertIsInstance(ret, bool) - def cpu_affinity(self, ret, proc): + def cpu_affinity(self, ret, info): self.assertIsInstance(ret, list) assert ret != [], ret - cpus = range(psutil.cpu_count()) + cpus = list(range(psutil.cpu_count())) for n in ret: self.assertIsInstance(n, int) self.assertIn(n, cpus) - def terminal(self, ret, proc): + def terminal(self, ret, info): self.assertIsInstance(ret, (str, type(None))) if ret is not None: assert os.path.isabs(ret), ret assert os.path.exists(ret), ret - def memory_maps(self, ret, proc): + def memory_maps(self, ret, info): for nt in ret: self.assertIsInstance(nt.addr, str) self.assertIsInstance(nt.perms, str) @@ -674,11 +698,11 @@ class TestFetchAllProcesses(unittest.TestCase): self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0) - def num_handles(self, ret, proc): + def num_handles(self, ret, info): self.assertIsInstance(ret, int) self.assertGreaterEqual(ret, 0) - def nice(self, ret, proc): + def nice(self, ret, info): self.assertIsInstance(ret, int) if POSIX: assert -20 <= ret <= 20, ret @@ -686,20 +710,24 @@ class TestFetchAllProcesses(unittest.TestCase): priorities = [getattr(psutil, x) for x in dir(psutil) if x.endswith('_PRIORITY_CLASS')] self.assertIn(ret, priorities) + if sys.version_info > (3, 4): + self.assertIsInstance(ret, enum.IntEnum) + else: + self.assertIsInstance(ret, int) - def num_ctx_switches(self, ret, proc): + def num_ctx_switches(self, ret, info): assert is_namedtuple(ret) for value in ret: self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0) - def rlimit(self, ret, proc): + def rlimit(self, ret, info): self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertGreaterEqual(ret[0], -1) self.assertGreaterEqual(ret[1], -1) - def environ(self, ret, proc): + def environ(self, ret, info): self.assertIsInstance(ret, dict) for k, v in ret.items(): self.assertIsInstance(k, str) @@ -707,5 +735,5 @@ class TestFetchAllProcesses(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 872ddf50..f12e1d29 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -17,7 +17,6 @@ import re import shutil import socket import struct -import tempfile import textwrap import time import warnings @@ -29,35 +28,44 @@ from psutil._compat import FileNotFoundError from psutil._compat import PY3 from psutil._compat import u from psutil.tests import call_until +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import GLOBAL_TIMEOUT from psutil.tests import HAS_BATTERY from psutil.tests import HAS_CPU_FREQ from psutil.tests import HAS_GETLOADAVG from psutil.tests import HAS_RLIMIT -from psutil.tests import MEMORY_TOLERANCE from psutil.tests import mock +from psutil.tests import PsutilTestCase from psutil.tests import PYPY -from psutil.tests import pyrun -from psutil.tests import reap_children from psutil.tests import reload_module from psutil.tests import retry_on_failure from psutil.tests import safe_rmpath from psutil.tests import sh from psutil.tests import skip_on_not_implemented -from psutil.tests import TESTFN from psutil.tests import ThreadTask -from psutil.tests import TRAVIS +from psutil.tests import TOLERANCE_DISK_USAGE +from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import unittest from psutil.tests import which +if LINUX: + from psutil._pslinux import calculate_avail_vmem + from psutil._pslinux import CLOCK_TICKS + from psutil._pslinux import open_binary + from psutil._pslinux import RootFsDeviceFinder + HERE = os.path.abspath(os.path.dirname(__file__)) SIOCGIFADDR = 0x8915 SIOCGIFCONF = 0x8912 SIOCGIFHWADDR = 0x8927 +SIOCGIFNETMASK = 0x891b +SIOCGIFBRDADDR = 0x8919 if LINUX: SECTOR_SIZE = 512 EMPTY_TEMPERATURES = not glob.glob('/sys/class/hwmon/hwmon*') + # ===================================================================== # --- utils # ===================================================================== @@ -76,6 +84,49 @@ def get_ipv4_address(ifname): struct.pack('256s', ifname))[20:24]) +def get_ipv4_netmask(ifname): + import fcntl + ifname = ifname[:15] + if PY3: + ifname = bytes(ifname, 'ascii') + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + with contextlib.closing(s): + return socket.inet_ntoa( + fcntl.ioctl(s.fileno(), + SIOCGIFNETMASK, + struct.pack('256s', ifname))[20:24]) + + +def get_ipv4_broadcast(ifname): + import fcntl + ifname = ifname[:15] + if PY3: + ifname = bytes(ifname, 'ascii') + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + with contextlib.closing(s): + return socket.inet_ntoa( + fcntl.ioctl(s.fileno(), + SIOCGIFBRDADDR, + struct.pack('256s', ifname))[20:24]) + + +def get_ipv6_address(ifname): + with open("/proc/net/if_inet6", 'rt') as f: + for line in f.readlines(): + fields = line.split() + if fields[-1] == ifname: + break + else: + raise ValueError("could not find interface %r" % ifname) + unformatted = fields[0] + groups = [] + for i in range(0, len(unformatted), 4): + groups.append(unformatted[i:i + 4]) + formatted = ":".join(groups) + packed = socket.inet_pton(socket.AF_INET6, formatted) + return socket.inet_ntop(socket.AF_INET6, packed) + + def get_mac_address(ifname): import fcntl ifname = ifname[:15] @@ -141,6 +192,8 @@ def vmstat(stat): def get_free_version_info(): out = sh("free -V").strip() + if 'UNKNOWN' in out: + raise unittest.SkipTest("can't determine free version") return tuple(map(int, out.split()[-1].split('.'))) @@ -190,7 +243,7 @@ def mock_open_exception(for_path, exc): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemVirtualMemory(unittest.TestCase): +class TestSystemVirtualMemory(PsutilTestCase): def test_total(self): # free_value = free_physmem().total @@ -198,55 +251,51 @@ class TestSystemVirtualMemory(unittest.TestCase): # self.assertEqual(free_value, psutil_value) vmstat_value = vmstat('total memory') * 1024 psutil_value = psutil.virtual_memory().total - self.assertAlmostEqual(vmstat_value, psutil_value) - - # Older versions of procps used slab memory to calculate used memory. - # This got changed in: - # https://gitlab.com/procps-ng/procps/commit/ - # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e - @unittest.skipIf(LINUX and get_free_version_info() < (3, 3, 12), - "old free version") + self.assertAlmostEqual( + vmstat_value, psutil_value, delta=TOLERANCE_SYS_MEM) + @retry_on_failure() def test_used(self): + # Older versions of procps used slab memory to calculate used memory. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + if get_free_version_info() < (3, 3, 12): + raise self.skipTest("old free version") free = free_physmem() free_value = free.used psutil_value = psutil.virtual_memory().used self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE, + free_value, psutil_value, delta=TOLERANCE_SYS_MEM, msg='%s %s \n%s' % (free_value, psutil_value, free.output)) - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @retry_on_failure() def test_free(self): vmstat_value = vmstat('free memory') * 1024 psutil_value = psutil.virtual_memory().free self.assertAlmostEqual( - vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) + vmstat_value, psutil_value, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_buffers(self): vmstat_value = vmstat('buffer memory') * 1024 psutil_value = psutil.virtual_memory().buffers self.assertAlmostEqual( - vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) + vmstat_value, psutil_value, delta=TOLERANCE_SYS_MEM) - # https://travis-ci.org/giampaolo/psutil/jobs/226719664 - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @retry_on_failure() def test_active(self): vmstat_value = vmstat('active memory') * 1024 psutil_value = psutil.virtual_memory().active self.assertAlmostEqual( - vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) + vmstat_value, psutil_value, delta=TOLERANCE_SYS_MEM) - # https://travis-ci.org/giampaolo/psutil/jobs/227242952 - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @retry_on_failure() def test_inactive(self): vmstat_value = vmstat('inactive memory') * 1024 psutil_value = psutil.virtual_memory().inactive self.assertAlmostEqual( - vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) + vmstat_value, psutil_value, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_shared(self): @@ -256,7 +305,7 @@ class TestSystemVirtualMemory(unittest.TestCase): raise unittest.SkipTest("free does not support 'shared' column") psutil_value = psutil.virtual_memory().shared self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE, + free_value, psutil_value, delta=TOLERANCE_SYS_MEM, msg='%s %s \n%s' % (free_value, psutil_value, free.output)) @retry_on_failure() @@ -271,7 +320,7 @@ class TestSystemVirtualMemory(unittest.TestCase): free_value = int(lines[1].split()[-1]) psutil_value = psutil.virtual_memory().available self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE, + free_value, psutil_value, delta=TOLERANCE_SYS_MEM, msg='%s %s \n%s' % (free_value, psutil_value, out)) def test_warnings_on_misses(self): @@ -317,9 +366,6 @@ class TestSystemVirtualMemory(unittest.TestCase): def test_avail_old_percent(self): # Make sure that our calculation of avail mem for old kernels # is off by max 15%. - from psutil._pslinux import calculate_avail_vmem - from psutil._pslinux import open_binary - mems = {} with open_binary('/proc/meminfo') as f: for line in f: @@ -496,7 +542,7 @@ class TestSystemVirtualMemory(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemSwapMemory(unittest.TestCase): +class TestSystemSwapMemory(PsutilTestCase): @staticmethod def meminfo_has_swap_info(): @@ -509,21 +555,21 @@ class TestSystemSwapMemory(unittest.TestCase): free_value = free_swap().total psutil_value = psutil.swap_memory().total return self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE) + free_value, psutil_value, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_used(self): free_value = free_swap().used psutil_value = psutil.swap_memory().used return self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE) + free_value, psutil_value, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_free(self): free_value = free_swap().free psutil_value = psutil.swap_memory().free return self.assertAlmostEqual( - free_value, psutil_value, delta=MEMORY_TOLERANCE) + free_value, psutil_value, delta=TOLERANCE_SYS_MEM) def test_missing_sin_sout(self): with mock.patch('psutil._common.open', create=True) as m: @@ -573,7 +619,7 @@ class TestSystemSwapMemory(unittest.TestCase): total *= unit_multiplier free *= unit_multiplier self.assertEqual(swap.total, total) - self.assertAlmostEqual(swap.free, free, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(swap.free, free, delta=TOLERANCE_SYS_MEM) def test_emulate_meminfo_has_no_metrics(self): # Emulate a case where /proc/meminfo provides no swap metrics @@ -590,9 +636,8 @@ class TestSystemSwapMemory(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemCPUTimes(unittest.TestCase): +class TestSystemCPUTimes(PsutilTestCase): - @unittest.skipIf(TRAVIS, "unknown failure on travis") def test_fields(self): fields = psutil.cpu_times()._fields kernel_ver = re.findall(r'\d+\.\d+\.\d+', os.uname()[2])[0] @@ -612,7 +657,7 @@ class TestSystemCPUTimes(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemCPUCountLogical(unittest.TestCase): +class TestSystemCPUCountLogical(PsutilTestCase): @unittest.skipIf(not os.path.exists("/sys/devices/system/cpu/online"), "/sys/devices/system/cpu/online does not exist") @@ -676,7 +721,7 @@ class TestSystemCPUCountLogical(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemCPUCountPhysical(unittest.TestCase): +class TestSystemCPUCountCores(PsutilTestCase): @unittest.skipIf(not which("lscpu"), "lscpu utility not available") def test_against_lscpu(self): @@ -688,18 +733,25 @@ class TestSystemCPUCountPhysical(unittest.TestCase): core_ids.add(fields[1]) self.assertEqual(psutil.cpu_count(logical=False), len(core_ids)) + def test_method_2(self): + meth_1 = psutil._pslinux.cpu_count_cores() + with mock.patch('glob.glob', return_value=[]) as m: + meth_2 = psutil._pslinux.cpu_count_cores() + assert m.called + if meth_1 is not None: + self.assertEqual(meth_1, meth_2) + def test_emulate_none(self): with mock.patch('glob.glob', return_value=[]) as m1: with mock.patch('psutil._common.open', create=True) as m2: - self.assertIsNone(psutil._pslinux.cpu_count_physical()) + self.assertIsNone(psutil._pslinux.cpu_count_cores()) assert m1.called assert m2.called @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemCPUFrequency(unittest.TestCase): +class TestSystemCPUFrequency(PsutilTestCase): - @unittest.skipIf(TRAVIS, "fails on Travis") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_emulate_use_second_file(self): # https://github.com/giampaolo/psutil/issues/981 @@ -722,18 +774,14 @@ class TestSystemCPUFrequency(unittest.TestCase): if path.startswith('/sys/devices/system/cpu/'): return False else: - if path == "/proc/cpuinfo": - flags.append(None) return os_path_exists(path) - flags = [] os_path_exists = os.path.exists try: with mock.patch("os.path.exists", side_effect=path_exists_mock): reload_module(psutil._pslinux) ret = psutil.cpu_freq() assert ret - assert flags self.assertEqual(ret.max, 0.0) self.assertEqual(ret.min, 0.0) for freq in psutil.cpu_freq(percpu=True): @@ -820,7 +868,6 @@ class TestSystemCPUFrequency(unittest.TestCase): if freq[1].max != 0.0: self.assertEqual(freq[1].max, 600.0) - @unittest.skipIf(TRAVIS, "fails on Travis") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_emulate_no_scaling_cur_freq_file(self): # See: https://github.com/giampaolo/psutil/issues/1071 @@ -845,15 +892,13 @@ class TestSystemCPUFrequency(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemCPUStats(unittest.TestCase): +class TestSystemCPUStats(PsutilTestCase): - @unittest.skipIf(TRAVIS, "fails on Travis") def test_ctx_switches(self): vmstat_value = vmstat("context switches") psutil_value = psutil.cpu_stats().ctx_switches self.assertAlmostEqual(vmstat_value, psutil_value, delta=500) - @unittest.skipIf(TRAVIS, "fails on Travis") def test_interrupts(self): vmstat_value = vmstat("interrupts") psutil_value = psutil.cpu_stats().interrupts @@ -861,7 +906,7 @@ class TestSystemCPUStats(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestLoadAvg(unittest.TestCase): +class TestLoadAvg(PsutilTestCase): @unittest.skipIf(not HAS_GETLOADAVG, "not supported") def test_getloadavg(self): @@ -880,7 +925,7 @@ class TestLoadAvg(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemNetIfAddrs(unittest.TestCase): +class TestSystemNetIfAddrs(PsutilTestCase): def test_ips(self): for name, addrs in psutil.net_if_addrs().items(): @@ -889,11 +934,24 @@ class TestSystemNetIfAddrs(unittest.TestCase): self.assertEqual(addr.address, get_mac_address(name)) elif addr.family == socket.AF_INET: self.assertEqual(addr.address, get_ipv4_address(name)) - # TODO: test for AF_INET6 family + self.assertEqual(addr.netmask, get_ipv4_netmask(name)) + if addr.broadcast is not None: + self.assertEqual(addr.broadcast, + get_ipv4_broadcast(name)) + else: + self.assertEqual(get_ipv4_broadcast(name), '0.0.0.0') + elif addr.family == socket.AF_INET6: + # IPv6 addresses can have a percent symbol at the end. + # E.g. these 2 are equivalent: + # "fe80::1ff:fe23:4567:890a" + # "fe80::1ff:fe23:4567:890a%eth0" + # That is the "zone id" portion, which usually is the name + # of the network interface. + address = addr.address.split('%')[0] + self.assertEqual(address, get_ipv6_address(name)) # XXX - not reliable when having virtual NICs installed by Docker. # @unittest.skipIf(not which('ip'), "'ip' utility not available") - # @unittest.skipIf(TRAVIS, "skipped on Travis") # def test_net_if_names(self): # out = sh("ip addr").strip() # nics = [x for x in psutil.net_if_addrs().keys() if ':' not in x] @@ -909,7 +967,7 @@ class TestSystemNetIfAddrs(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemNetIfStats(unittest.TestCase): +class TestSystemNetIfStats(PsutilTestCase): def test_against_ifconfig(self): for name, stats in psutil.net_if_stats().items(): @@ -918,14 +976,18 @@ class TestSystemNetIfStats(unittest.TestCase): except RuntimeError: pass else: - # Not always reliable. - # self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) + self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) self.assertEqual(stats.mtu, int(re.findall(r'(?i)MTU[: ](\d+)', out)[0])) + def test_mtu(self): + for name, stats in psutil.net_if_stats().items(): + with open("/sys/class/net/%s/mtu" % name, "rt") as f: + self.assertEqual(stats.mtu, int(f.read().strip())) + @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemNetIOCounters(unittest.TestCase): +class TestSystemNetIOCounters(PsutilTestCase): @retry_on_failure() def test_against_ifconfig(self): @@ -971,7 +1033,7 @@ class TestSystemNetIOCounters(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemNetConnections(unittest.TestCase): +class TestSystemNetConnections(PsutilTestCase): @mock.patch('psutil._pslinux.socket.inet_ntop', side_effect=ValueError) @mock.patch('psutil._pslinux.supports_ipv6', return_value=False) @@ -1004,7 +1066,7 @@ class TestSystemNetConnections(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemDiskPartitions(unittest.TestCase): +class TestSystemDiskPartitions(PsutilTestCase): @unittest.skipIf(not hasattr(os, 'statvfs'), "os.statvfs() not available") @skip_on_not_implemented() @@ -1026,11 +1088,10 @@ class TestSystemDiskPartitions(unittest.TestCase): usage = psutil.disk_usage(part.mountpoint) dev, total, used, free = df(part.mountpoint) self.assertEqual(usage.total, total) - # 10 MB tollerance - if abs(usage.free - free) > 10 * 1024 * 1024: - self.fail("psutil=%s, df=%s" % (usage.free, free)) - if abs(usage.used - used) > 10 * 1024 * 1024: - self.fail("psutil=%s, df=%s" % (usage.used, used)) + self.assertAlmostEqual(usage.free, free, + delta=TOLERANCE_DISK_USAGE) + self.assertAlmostEqual(usage.used, used, + delta=TOLERANCE_DISK_USAGE) def test_zfs_fs(self): # Test that ZFS partitions are returned. @@ -1069,7 +1130,7 @@ class TestSystemDiskPartitions(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSystemDiskIoCounters(unittest.TestCase): +class TestSystemDiskIoCounters(PsutilTestCase): def test_emulate_kernel_2_4(self): # Tests /proc/diskstats parsing format for 2.4 kernels, see: @@ -1218,13 +1279,74 @@ class TestDiskSwaps(unittest.TestCase): tuple(swaps[i])) +class TestRootFsDeviceFinder(PsutilTestCase): + + def setUp(self): + dev = os.stat("/").st_dev + self.major = os.major(dev) + self.minor = os.minor(dev) + + def test_call_methods(self): + finder = RootFsDeviceFinder() + if os.path.exists("/proc/partitions"): + finder.ask_proc_partitions() + else: + self.assertRaises(FileNotFoundError, finder.ask_proc_partitions) + if os.path.exists("/sys/dev/block/%s:%s/uevent" % ( + self.major, self.minor)): + finder.ask_sys_dev_block() + else: + self.assertRaises(FileNotFoundError, finder.ask_sys_dev_block) + finder.ask_sys_class_block() + + @unittest.skipIf(GITHUB_ACTIONS, "unsupported on GITHUB_ACTIONS") + def test_comparisons(self): + finder = RootFsDeviceFinder() + self.assertIsNotNone(finder.find()) + + a = b = c = None + if os.path.exists("/proc/partitions"): + a = finder.ask_proc_partitions() + if os.path.exists("/sys/dev/block/%s:%s/uevent" % ( + self.major, self.minor)): + b = finder.ask_sys_class_block() + c = finder.ask_sys_dev_block() + + base = a or b or c + if base and a: + self.assertEqual(base, a) + if base and b: + self.assertEqual(base, b) + if base and c: + self.assertEqual(base, c) + + @unittest.skipIf(not which("findmnt"), "findmnt utility not available") + @unittest.skipIf(GITHUB_ACTIONS, "unsupported on GITHUB_ACTIONS") + def test_against_findmnt(self): + psutil_value = RootFsDeviceFinder().find() + findmnt_value = sh("findmnt -o SOURCE -rn /") + self.assertEqual(psutil_value, findmnt_value) + + def test_disk_partitions_mocked(self): + with mock.patch( + 'psutil._pslinux.cext.disk_partitions', + return_value=[('/dev/root', '/', 'ext4', 'rw')]) as m: + part = psutil.disk_partitions()[0] + assert m.called + if not GITHUB_ACTIONS: + self.assertNotEqual(part.device, "/dev/root") + self.assertEqual(part.device, RootFsDeviceFinder().find()) + else: + self.assertEqual(part.device, "/dev/root") + + # ===================================================================== # --- misc # ===================================================================== @unittest.skipIf(not LINUX, "LINUX only") -class TestMisc(unittest.TestCase): +class TestMisc(PsutilTestCase): def test_boot_time(self): vmstat_value = vmstat('boot time') @@ -1232,7 +1354,8 @@ class TestMisc(unittest.TestCase): self.assertEqual(int(vmstat_value), int(psutil_value)) def test_no_procfs_on_import(self): - my_procfs = tempfile.mkdtemp() + my_procfs = self.get_testfn() + os.mkdir(my_procfs) with open(os.path.join(my_procfs, 'stat'), 'w') as f: f.write('cpu 0 0 0 0 0 0 0 0 0 0\n') @@ -1360,7 +1483,8 @@ class TestMisc(unittest.TestCase): assert m.called def test_procfs_path(self): - tdir = tempfile.mkdtemp() + tdir = self.get_testfn() + os.mkdir(tdir) try: psutil.PROCFS_PATH = tdir self.assertRaises(IOError, psutil.virtual_memory) @@ -1376,25 +1500,23 @@ class TestMisc(unittest.TestCase): self.assertRaises(psutil.NoSuchProcess, psutil.Process) finally: psutil.PROCFS_PATH = "/proc" - os.rmdir(tdir) + @retry_on_failure() def test_issue_687(self): # In case of thread ID: # - pid_exists() is supposed to return False # - Process(tid) is supposed to work # - pids() should not return the TID # See: https://github.com/giampaolo/psutil/issues/687 - t = ThreadTask() - t.start() - try: + with ThreadTask(): p = psutil.Process() - tid = p.threads()[1].id - assert not psutil.pid_exists(tid), tid + threads = p.threads() + self.assertEqual(len(threads), 2) + tid = sorted(threads, key=lambda x: x.id)[1].id + self.assertNotEqual(p.pid, tid) pt = psutil.Process(tid) pt.as_dict() self.assertNotIn(tid, psutil.pids()) - finally: - t.stop() def test_pid_exists_no_proc_status(self): # Internally pid_exists relies on /proc/{pid}/status. @@ -1412,7 +1534,7 @@ class TestMisc(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") @unittest.skipIf(not HAS_BATTERY, "no battery") -class TestSensorsBattery(unittest.TestCase): +class TestSensorsBattery(PsutilTestCase): @unittest.skipIf(not which("acpi"), "acpi utility not available") def test_percent(self): @@ -1421,17 +1543,6 @@ class TestSensorsBattery(unittest.TestCase): psutil_value = psutil.sensors_battery().percent self.assertAlmostEqual(acpi_value, psutil_value, delta=1) - @unittest.skipIf(not which("acpi"), "acpi utility not available") - def test_power_plugged(self): - out = sh("acpi -b") - if 'unknown' in out.lower(): - return unittest.skip("acpi output not reliable") - if 'discharging at zero rate' in out: - plugged = True - else: - plugged = "Charging" in out.split('\n')[0] - self.assertEqual(psutil.sensors_battery().power_plugged, plugged) - def test_emulate_power_plugged(self): # Pretend the AC power cable is connected. def open_mock(name, *args, **kwargs): @@ -1514,17 +1625,6 @@ class TestSensorsBattery(unittest.TestCase): self.assertIsNone(psutil.sensors_battery().power_plugged) assert m.called - def test_emulate_no_base_files(self): - # Emulate a case where base metrics files are not present, - # in which case we're supposed to get None. - with mock_open_exception( - "/sys/class/power_supply/BAT0/energy_now", - IOError(errno.ENOENT, "")): - with mock_open_exception( - "/sys/class/power_supply/BAT0/charge_now", - IOError(errno.ENOENT, "")): - self.assertIsNone(psutil.sensors_battery()) - def test_emulate_energy_full_0(self): # Emulate a case where energy_full files returns 0. with mock_open_content( @@ -1560,7 +1660,30 @@ class TestSensorsBattery(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSensorsTemperatures(unittest.TestCase): +class TestSensorsBatteryEmulated(PsutilTestCase): + + def test_it(self): + def open_mock(name, *args, **kwargs): + if name.endswith("/energy_now"): + return io.StringIO(u("60000000")) + elif name.endswith("/power_now"): + return io.StringIO(u("0")) + elif name.endswith("/energy_full"): + return io.StringIO(u("60000001")) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch('os.listdir', return_value=["BAT0"]) as mlistdir: + with mock.patch(patch_point, side_effect=open_mock) as mopen: + self.assertIsNotNone(psutil.sensors_battery()) + assert mlistdir.called + assert mopen.called + + +@unittest.skipIf(not LINUX, "LINUX only") +class TestSensorsTemperatures(PsutilTestCase): def test_emulate_class_hwmon(self): def open_mock(name, *args, **kwargs): @@ -1626,7 +1749,7 @@ class TestSensorsTemperatures(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestSensorsFans(unittest.TestCase): +class TestSensorsFans(PsutilTestCase): def test_emulate_data(self): def open_mock(name, *args, **kwargs): @@ -1655,22 +1778,18 @@ class TestSensorsFans(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestProcess(unittest.TestCase): - - def setUp(self): - safe_rmpath(TESTFN) - - tearDown = setUp +class TestProcess(PsutilTestCase): + @retry_on_failure() def test_memory_full_info(self): + testfn = self.get_testfn() src = textwrap.dedent(""" import time with open("%s", "w") as f: time.sleep(10) - """ % TESTFN) - sproc = pyrun(src) - self.addCleanup(reap_children) - call_until(lambda: os.listdir('.'), "'%s' not in ret" % TESTFN) + """ % testfn) + sproc = self.pyrun(src) + call_until(lambda: os.listdir('.'), "'%s' not in ret" % testfn) p = psutil.Process(sproc.pid) time.sleep(.1) mem = p.memory_full_info() @@ -1720,46 +1839,47 @@ class TestProcess(unittest.TestCase): # On PYPY file descriptors are not closed fast enough. @unittest.skipIf(PYPY, "unreliable on PYPY") def test_open_files_mode(self): - def get_test_file(): + def get_test_file(fname): p = psutil.Process() - giveup_at = time.time() + 2 + giveup_at = time.time() + GLOBAL_TIMEOUT while True: for file in p.open_files(): - if file.path == os.path.abspath(TESTFN): + if file.path == os.path.abspath(fname): return file elif time.time() > giveup_at: break raise RuntimeError("timeout looking for test file") # - with open(TESTFN, "w"): - self.assertEqual(get_test_file().mode, "w") - with open(TESTFN, "r"): - self.assertEqual(get_test_file().mode, "r") - with open(TESTFN, "a"): - self.assertEqual(get_test_file().mode, "a") + testfn = self.get_testfn() + with open(testfn, "w"): + self.assertEqual(get_test_file(testfn).mode, "w") + with open(testfn, "r"): + self.assertEqual(get_test_file(testfn).mode, "r") + with open(testfn, "a"): + self.assertEqual(get_test_file(testfn).mode, "a") # - with open(TESTFN, "r+"): - self.assertEqual(get_test_file().mode, "r+") - with open(TESTFN, "w+"): - self.assertEqual(get_test_file().mode, "r+") - with open(TESTFN, "a+"): - self.assertEqual(get_test_file().mode, "a+") + with open(testfn, "r+"): + self.assertEqual(get_test_file(testfn).mode, "r+") + with open(testfn, "w+"): + self.assertEqual(get_test_file(testfn).mode, "r+") + with open(testfn, "a+"): + self.assertEqual(get_test_file(testfn).mode, "a+") # note: "x" bit is not supported if PY3: - safe_rmpath(TESTFN) - with open(TESTFN, "x"): - self.assertEqual(get_test_file().mode, "w") - safe_rmpath(TESTFN) - with open(TESTFN, "x+"): - self.assertEqual(get_test_file().mode, "r+") + safe_rmpath(testfn) + with open(testfn, "x"): + self.assertEqual(get_test_file(testfn).mode, "w") + safe_rmpath(testfn) + with open(testfn, "x+"): + self.assertEqual(get_test_file(testfn).mode, "r+") def test_open_files_file_gone(self): # simulates a file which gets deleted during open_files() # execution p = psutil.Process() files = p.open_files() - with tempfile.NamedTemporaryFile(): + with open(self.get_testfn(), 'w'): # give the kernel some time to see the new file call_until(p.open_files, "len(ret) != %i" % len(files)) with mock.patch('psutil._pslinux.os.readlink', @@ -1780,7 +1900,7 @@ class TestProcess(unittest.TestCase): # https://travis-ci.org/giampaolo/psutil/jobs/225694530 p = psutil.Process() files = p.open_files() - with tempfile.NamedTemporaryFile(): + with open(self.get_testfn(), 'w'): # give the kernel some time to see the new file call_until(p.open_files, "len(ret) != %i" % len(files)) patch_point = 'builtins.open' if PY3 else '__builtin__.open' @@ -1790,6 +1910,22 @@ class TestProcess(unittest.TestCase): assert not files assert m.called + def test_open_files_enametoolong(self): + # Simulate a case where /proc/{pid}/fd/{fd} symlink + # points to a file with full path longer than PATH_MAX, see: + # https://github.com/giampaolo/psutil/issues/1940 + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(p.open_files, "len(ret) != %i" % len(files)) + patch_point = 'psutil._pslinux.os.readlink' + with mock.patch(patch_point, + side_effect=OSError(errno.ENAMETOOLONG, "")) as m: + files = p.open_files() + assert not files + assert m.called + # --- mocked tests def test_terminal_mocked(self): @@ -1916,7 +2052,7 @@ class TestProcess(unittest.TestCase): # Emulate a case where rlimit() raises ENOSYS, which may # happen in case of zombie process: # https://travis-ci.org/giampaolo/psutil/jobs/51368273 - with mock.patch("psutil._pslinux.cext.linux_prlimit", + with mock.patch("psutil._pslinux.prlimit", side_effect=OSError(errno.ENOSYS, "")) as m: p = psutil.Process() p.name() @@ -1938,8 +2074,6 @@ class TestProcess(unittest.TestCase): self.assertEqual(exc.exception.name, p.name()) def test_stat_file_parsing(self): - from psutil._pslinux import CLOCK_TICKS - args = [ "0", # pid "(cat)", # name @@ -2025,9 +2159,19 @@ class TestProcess(unittest.TestCase): self.assertEqual(gids.saved, 1006) self.assertEqual(p._proc._get_eligible_cpus(), list(range(0, 8))) + def test_connections_enametoolong(self): + # Simulate a case where /proc/{pid}/fd/{fd} symlink points to + # a file with full path longer than PATH_MAX, see: + # https://github.com/giampaolo/psutil/issues/1940 + with mock.patch('psutil._pslinux.os.readlink', + side_effect=OSError(errno.ENAMETOOLONG, "")) as m: + p = psutil.Process() + assert not p.connections() + assert m.called + @unittest.skipIf(not LINUX, "LINUX only") -class TestProcessAgainstStatus(unittest.TestCase): +class TestProcessAgainstStatus(PsutilTestCase): """/proc/pid/stat and /proc/pid/status have many values in common. Whenever possible, psutil uses /proc/pid/stat (it's faster). For all those cases we check that the value found in @@ -2110,7 +2254,7 @@ class TestProcessAgainstStatus(unittest.TestCase): @unittest.skipIf(not LINUX, "LINUX only") -class TestUtils(unittest.TestCase): +class TestUtils(PsutilTestCase): def test_readlink(self): with mock.patch("os.readlink", return_value="foo (deleted)") as m: @@ -2118,15 +2262,15 @@ class TestUtils(unittest.TestCase): assert m.called def test_cat(self): - fname = os.path.abspath(TESTFN) - with open(fname, "wt") as f: + testfn = self.get_testfn() + with open(testfn, "wt") as f: f.write("foo ") - self.assertEqual(psutil._psplatform.cat(TESTFN, binary=False), "foo") - self.assertEqual(psutil._psplatform.cat(TESTFN, binary=True), b"foo") + self.assertEqual(psutil._psplatform.cat(testfn, binary=False), "foo") + self.assertEqual(psutil._psplatform.cat(testfn, binary=True), b"foo") self.assertEqual( - psutil._psplatform.cat(TESTFN + '??', fallback="bar"), "bar") + psutil._psplatform.cat(testfn + '??', fallback="bar"), "bar") if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memleaks.py index 4da9fea5..ecec8462 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memleaks.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -17,27 +17,20 @@ because of how its JIT handles memory, so tests are skipped. from __future__ import print_function import functools -import gc import os -import sys -import threading -import time import psutil import psutil._common -from psutil import FREEBSD from psutil import LINUX from psutil import MACOS from psutil import OPENBSD from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS -from psutil._common import bytes2human from psutil._compat import ProcessLookupError -from psutil._compat import xrange -from psutil.tests import CIRRUS +from psutil._compat import super from psutil.tests import create_sockets -from psutil.tests import get_test_subprocess +from psutil.tests import get_testfn from psutil.tests import HAS_CPU_AFFINITY from psutil.tests import HAS_CPU_FREQ from psutil.tests import HAS_DISK_SWAPS @@ -51,128 +44,38 @@ from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES -from psutil.tests import PYPY -from psutil.tests import reap_children -from psutil.tests import safe_rmpath +from psutil.tests import process_namespace from psutil.tests import skip_on_access_denied -from psutil.tests import TESTFN -from psutil.tests import TRAVIS +from psutil.tests import spawn_testproc +from psutil.tests import system_namespace +from psutil.tests import terminate +from psutil.tests import TestMemoryLeak from psutil.tests import unittest -# configurable opts -LOOPS = 1000 -MEMORY_TOLERANCE = 4096 -RETRY_FOR = 3 -SKIP_PYTHON_IMPL = True - cext = psutil._psplatform.cext thisproc = psutil.Process() +FEW_TIMES = 5 -# =================================================================== -# utils -# =================================================================== - - -def skip_if_linux(): - return unittest.skipIf(LINUX and SKIP_PYTHON_IMPL, - "worthless on LINUX (pure python)") - - -@unittest.skipIf(PYPY, "unreliable on PYPY") -class TestMemLeak(unittest.TestCase): - """Base framework class which calls a function many times and - produces a failure if process memory usage keeps increasing - between calls or over time. +def fewtimes_if_linux(): + """Decorator for those Linux functions which are implemented in pure + Python, and which we want to run faster. """ - tolerance = MEMORY_TOLERANCE - loops = LOOPS - retry_for = RETRY_FOR - - def setUp(self): - gc.collect() - - def execute(self, fun, *args, **kwargs): - """Test a callable.""" - def call_many_times(): - for x in xrange(loops): - self._call(fun, *args, **kwargs) - del x - gc.collect() - - tolerance = kwargs.pop('tolerance_', None) or self.tolerance - loops = kwargs.pop('loops_', None) or self.loops - retry_for = kwargs.pop('retry_for_', None) or self.retry_for - - # warm up - for x in range(10): - self._call(fun, *args, **kwargs) - self.assertEqual(gc.garbage, []) - self.assertEqual(threading.active_count(), 1) - self.assertEqual(thisproc.children(), []) - - # Get 2 distinct memory samples, before and after having - # called fun repeadetly. - # step 1 - call_many_times() - mem1 = self._get_mem() - # step 2 - call_many_times() - mem2 = self._get_mem() - - diff1 = mem2 - mem1 - if diff1 > tolerance: - # This doesn't necessarily mean we have a leak yet. - # At this point we assume that after having called the - # function so many times the memory usage is stabilized - # and if there are no leaks it should not increase - # anymore. - # Let's keep calling fun for 3 more seconds and fail if - # we notice any difference. - ncalls = 0 - stop_at = time.time() + retry_for - while time.time() <= stop_at: - self._call(fun, *args, **kwargs) - ncalls += 1 - - del stop_at - gc.collect() - mem3 = self._get_mem() - diff2 = mem3 - mem2 - - if mem3 > mem2: - # failure - extra_proc_mem = bytes2human(diff1 + diff2) - print("exta proc mem: %s" % extra_proc_mem, file=sys.stderr) - msg = "+%s after %s calls, +%s after another %s calls, " - msg += "+%s extra proc mem" - msg = msg % ( - bytes2human(diff1), loops, bytes2human(diff2), ncalls, - extra_proc_mem) - self.fail(msg) - - def execute_w_exc(self, exc, fun, *args, **kwargs): - """Convenience function which tests a callable raising - an exception. - """ - def call(): - self.assertRaises(exc, fun, *args, **kwargs) - - self.execute(call) - - @staticmethod - def _get_mem(): - # By using USS memory it seems it's less likely to bump - # into false positives. - if LINUX or WINDOWS or MACOS: - return thisproc.memory_full_info().uss - else: - return thisproc.memory_info().rss - - @staticmethod - def _call(fun, *args, **kwargs): - fun(*args, **kwargs) + def decorator(fun): + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + if LINUX: + before = self.__class__.times + try: + self.__class__.times = FEW_TIMES + return fun(self, *args, **kwargs) + finally: + self.__class__.times = before + else: + return fun(self, *args, **kwargs) + return wrapper + return decorator # =================================================================== @@ -180,89 +83,82 @@ class TestMemLeak(unittest.TestCase): # =================================================================== -class TestProcessObjectLeaks(TestMemLeak): +class TestProcessObjectLeaks(TestMemoryLeak): """Test leaks of Process class methods.""" proc = thisproc def test_coverage(self): - skip = set(( - "pid", "as_dict", "children", "cpu_affinity", "cpu_percent", - "ionice", "is_running", "kill", "memory_info_ex", "memory_percent", - "nice", "oneshot", "parent", "parents", "rlimit", "send_signal", - "suspend", "terminate", "wait")) - for name in dir(psutil.Process): - if name.startswith('_'): - continue - if name in skip: - continue - self.assertTrue(hasattr(self, "test_" + name), msg=name) - - @skip_if_linux() + ns = process_namespace(None) + ns.test_class_coverage(self, ns.getters + ns.setters) + + @fewtimes_if_linux() def test_name(self): self.execute(self.proc.name) - @skip_if_linux() + @fewtimes_if_linux() def test_cmdline(self): self.execute(self.proc.cmdline) - @skip_if_linux() + @fewtimes_if_linux() def test_exe(self): self.execute(self.proc.exe) - @skip_if_linux() + @fewtimes_if_linux() def test_ppid(self): self.execute(self.proc.ppid) @unittest.skipIf(not POSIX, "POSIX only") - @skip_if_linux() + @fewtimes_if_linux() def test_uids(self): self.execute(self.proc.uids) @unittest.skipIf(not POSIX, "POSIX only") - @skip_if_linux() + @fewtimes_if_linux() def test_gids(self): self.execute(self.proc.gids) - @skip_if_linux() + @fewtimes_if_linux() def test_status(self): self.execute(self.proc.status) - def test_nice_get(self): + def test_nice(self): self.execute(self.proc.nice) def test_nice_set(self): niceness = thisproc.nice() - self.execute(self.proc.nice, niceness) + self.execute(lambda: self.proc.nice(niceness)) @unittest.skipIf(not HAS_IONICE, "not supported") - def test_ionice_get(self): + def test_ionice(self): self.execute(self.proc.ionice) @unittest.skipIf(not HAS_IONICE, "not supported") def test_ionice_set(self): if WINDOWS: value = thisproc.ionice() - self.execute(self.proc.ionice, value) + self.execute(lambda: self.proc.ionice(value)) else: - self.execute(self.proc.ionice, psutil.IOPRIO_CLASS_NONE) + self.execute(lambda: self.proc.ionice(psutil.IOPRIO_CLASS_NONE)) fun = functools.partial(cext.proc_ioprio_set, os.getpid(), -1, 0) self.execute_w_exc(OSError, fun) @unittest.skipIf(not HAS_PROC_IO_COUNTERS, "not supported") - @skip_if_linux() + @fewtimes_if_linux() def test_io_counters(self): self.execute(self.proc.io_counters) @unittest.skipIf(POSIX, "worthless on POSIX") def test_username(self): + # always open 1 handle on Windows (only once) + psutil.Process().username() self.execute(self.proc.username) - @skip_if_linux() + @fewtimes_if_linux() def test_create_time(self): self.execute(self.proc.create_time) - @skip_if_linux() + @fewtimes_if_linux() @skip_on_access_denied(only_if=OPENBSD) def test_num_threads(self): self.execute(self.proc.num_threads) @@ -272,85 +168,83 @@ class TestProcessObjectLeaks(TestMemLeak): self.execute(self.proc.num_handles) @unittest.skipIf(not POSIX, "POSIX only") - @skip_if_linux() + @fewtimes_if_linux() def test_num_fds(self): self.execute(self.proc.num_fds) - @skip_if_linux() + @fewtimes_if_linux() def test_num_ctx_switches(self): self.execute(self.proc.num_ctx_switches) - @skip_if_linux() + @fewtimes_if_linux() @skip_on_access_denied(only_if=OPENBSD) def test_threads(self): self.execute(self.proc.threads) - @skip_if_linux() + @fewtimes_if_linux() def test_cpu_times(self): self.execute(self.proc.cpu_times) - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(not HAS_PROC_CPU_NUM, "not supported") def test_cpu_num(self): self.execute(self.proc.cpu_num) - @skip_if_linux() + @fewtimes_if_linux() def test_memory_info(self): self.execute(self.proc.memory_info) - @skip_if_linux() + @fewtimes_if_linux() def test_memory_full_info(self): self.execute(self.proc.memory_full_info) @unittest.skipIf(not POSIX, "POSIX only") - @skip_if_linux() + @fewtimes_if_linux() def test_terminal(self): self.execute(self.proc.terminal) - @unittest.skipIf(POSIX and SKIP_PYTHON_IMPL, - "worthless on POSIX (pure python)") def test_resume(self): - self.execute(self.proc.resume) + times = FEW_TIMES if POSIX else self.times + self.execute(self.proc.resume, times=times) - @skip_if_linux() + @fewtimes_if_linux() def test_cwd(self): self.execute(self.proc.cwd) @unittest.skipIf(not HAS_CPU_AFFINITY, "not supported") - def test_cpu_affinity_get(self): + def test_cpu_affinity(self): self.execute(self.proc.cpu_affinity) @unittest.skipIf(not HAS_CPU_AFFINITY, "not supported") def test_cpu_affinity_set(self): affinity = thisproc.cpu_affinity() - self.execute(self.proc.cpu_affinity, affinity) - if not TRAVIS: - self.execute_w_exc(ValueError, self.proc.cpu_affinity, [-1]) + self.execute(lambda: self.proc.cpu_affinity(affinity)) + self.execute_w_exc( + ValueError, lambda: self.proc.cpu_affinity([-1])) - @skip_if_linux() + @fewtimes_if_linux() def test_open_files(self): - safe_rmpath(TESTFN) # needed after UNIX socket test has run - with open(TESTFN, 'w'): + with open(get_testfn(), 'w'): self.execute(self.proc.open_files) @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") - @skip_if_linux() + @fewtimes_if_linux() def test_memory_maps(self): self.execute(self.proc.memory_maps) @unittest.skipIf(not LINUX, "LINUX only") @unittest.skipIf(not HAS_RLIMIT, "not supported") - def test_rlimit_get(self): - self.execute(self.proc.rlimit, psutil.RLIMIT_NOFILE) + def test_rlimit(self): + self.execute(lambda: self.proc.rlimit(psutil.RLIMIT_NOFILE)) @unittest.skipIf(not LINUX, "LINUX only") @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_set(self): limit = thisproc.rlimit(psutil.RLIMIT_NOFILE) - self.execute(self.proc.rlimit, psutil.RLIMIT_NOFILE, limit) - self.execute_w_exc(OSError, self.proc.rlimit, -1) + self.execute(lambda: self.proc.rlimit(psutil.RLIMIT_NOFILE, limit)) + self.execute_w_exc((OSError, ValueError), lambda: self.proc.rlimit(-1)) - @skip_if_linux() + @fewtimes_if_linux() # Windows implementation is based on a single system-wide # function (tested later). @unittest.skipIf(WINDOWS, "worthless on WINDOWS") @@ -360,7 +254,7 @@ class TestProcessObjectLeaks(TestMemLeak): # be executed. with create_sockets(): kind = 'inet' if SUNOS else 'all' - self.execute(self.proc.connections, kind) + self.execute(lambda: self.proc.connections(kind)) @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): @@ -368,17 +262,7 @@ class TestProcessObjectLeaks(TestMemLeak): @unittest.skipIf(not WINDOWS, "WINDOWS only") def test_proc_info(self): - self.execute(cext.proc_info, os.getpid()) - - -@unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestProcessDualImplementation(TestMemLeak): - - def test_cmdline_peb_true(self): - self.execute(cext.proc_cmdline, os.getpid(), use_peb=True) - - def test_cmdline_peb_false(self): - self.execute(cext.proc_cmdline, os.getpid(), use_peb=False) + self.execute(lambda: cext.proc_info(os.getpid())) class TestTerminatedProcessLeaks(TestProcessObjectLeaks): @@ -390,20 +274,20 @@ class TestTerminatedProcessLeaks(TestProcessObjectLeaks): @classmethod def setUpClass(cls): - super(TestTerminatedProcessLeaks, cls).setUpClass() - p = get_test_subprocess() - cls.proc = psutil.Process(p.pid) + super().setUpClass() + cls.subp = spawn_testproc() + cls.proc = psutil.Process(cls.subp.pid) cls.proc.kill() cls.proc.wait() @classmethod def tearDownClass(cls): - super(TestTerminatedProcessLeaks, cls).tearDownClass() - reap_children() + super().tearDownClass() + terminate(cls.subp) - def _call(self, fun, *args, **kwargs): + def call(self, fun): try: - fun(*args, **kwargs) + fun() except psutil.NoSuchProcess: pass @@ -435,54 +319,58 @@ class TestTerminatedProcessLeaks(TestProcessObjectLeaks): self.execute(call) +@unittest.skipIf(not WINDOWS, "WINDOWS only") +class TestProcessDualImplementation(TestMemoryLeak): + + def test_cmdline_peb_true(self): + self.execute(lambda: cext.proc_cmdline(os.getpid(), use_peb=True)) + + def test_cmdline_peb_false(self): + self.execute(lambda: cext.proc_cmdline(os.getpid(), use_peb=False)) + + # =================================================================== # system APIs # =================================================================== -class TestModuleFunctionsLeaks(TestMemLeak): +class TestModuleFunctionsLeaks(TestMemoryLeak): """Test leaks of psutil module functions.""" def test_coverage(self): - skip = set(( - "version_info", "__version__", "process_iter", "wait_procs", - "cpu_percent", "cpu_times_percent", "cpu_count")) - for name in psutil.__all__: - if not name.islower(): - continue - if name in skip: - continue - self.assertTrue(hasattr(self, "test_" + name), msg=name) + ns = system_namespace() + ns.test_class_coverage(self, ns.all) # --- cpu - @skip_if_linux() - def test_cpu_count_logical(self): - self.execute(psutil.cpu_count, logical=True) + @fewtimes_if_linux() + def test_cpu_count(self): # logical + self.execute(lambda: psutil.cpu_count(logical=True)) - @skip_if_linux() - def test_cpu_count_physical(self): - self.execute(psutil.cpu_count, logical=False) + @fewtimes_if_linux() + def test_cpu_count_cores(self): + self.execute(lambda: psutil.cpu_count(logical=False)) - @skip_if_linux() + @fewtimes_if_linux() def test_cpu_times(self): self.execute(psutil.cpu_times) - @skip_if_linux() + @fewtimes_if_linux() def test_per_cpu_times(self): - self.execute(psutil.cpu_times, percpu=True) + self.execute(lambda: psutil.cpu_times(percpu=True)) - @skip_if_linux() + @fewtimes_if_linux() def test_cpu_stats(self): self.execute(psutil.cpu_stats) - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): self.execute(psutil.cpu_freq) @unittest.skipIf(not WINDOWS, "WINDOWS only") def test_getloadavg(self): + psutil.getloadavg() self.execute(psutil.getloadavg) # --- mem @@ -495,26 +383,24 @@ class TestModuleFunctionsLeaks(TestMemLeak): def test_swap_memory(self): self.execute(psutil.swap_memory) - @unittest.skipIf(POSIX and SKIP_PYTHON_IMPL, - "worthless on POSIX (pure python)") def test_pid_exists(self): - self.execute(psutil.pid_exists, os.getpid()) + times = FEW_TIMES if POSIX else self.times + self.execute(lambda: psutil.pid_exists(os.getpid()), times=times) # --- disk - @unittest.skipIf(POSIX and SKIP_PYTHON_IMPL, - "worthless on POSIX (pure python)") def test_disk_usage(self): - self.execute(psutil.disk_usage, '.') + times = FEW_TIMES if POSIX else self.times + self.execute(lambda: psutil.disk_usage('.'), times=times) def test_disk_partitions(self): self.execute(psutil.disk_partitions) @unittest.skipIf(LINUX and not os.path.exists('/proc/diskstats'), '/proc/diskstats not available on this Linux version') - @skip_if_linux() + @fewtimes_if_linux() def test_disk_io_counters(self): - self.execute(psutil.disk_io_counters, nowrap=False) + self.execute(lambda: psutil.disk_io_counters(nowrap=False)) @unittest.skipIf(not HAS_DISK_SWAPS, "not supported") def test_disk_swaps(self): @@ -522,61 +408,62 @@ class TestModuleFunctionsLeaks(TestMemLeak): # --- proc - @skip_if_linux() + @fewtimes_if_linux() def test_pids(self): self.execute(psutil.pids) # --- net - @unittest.skipIf(TRAVIS and MACOS, "false positive on TRAVIS + MACOS") - @unittest.skipIf(CIRRUS and FREEBSD, "false positive on CIRRUS + FREEBSD") - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(not HAS_NET_IO_COUNTERS, 'not supported') def test_net_io_counters(self): - self.execute(psutil.net_io_counters, nowrap=False) + self.execute(lambda: psutil.net_io_counters(nowrap=False)) - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(MACOS and os.getuid() != 0, "need root access") def test_net_connections(self): + # always opens and handle on Windows() (once) + psutil.net_connections(kind='all') with create_sockets(): - self.execute(psutil.net_connections) + self.execute(lambda: psutil.net_connections(kind='all')) def test_net_if_addrs(self): # Note: verified that on Windows this was a false positive. - self.execute(psutil.net_if_addrs, - tolerance_=80 * 1024 if WINDOWS else None) + tolerance = 80 * 1024 if WINDOWS else self.tolerance + self.execute(psutil.net_if_addrs, tolerance=tolerance) - @unittest.skipIf(TRAVIS, "EPERM on travis") def test_net_if_stats(self): self.execute(psutil.net_if_stats) # --- sensors - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") def test_sensors_battery(self): self.execute(psutil.sensors_battery) - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_sensors_temperatures(self): self.execute(psutil.sensors_temperatures) - @skip_if_linux() + @fewtimes_if_linux() @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") def test_sensors_fans(self): self.execute(psutil.sensors_fans) # --- others - @skip_if_linux() + @fewtimes_if_linux() def test_boot_time(self): self.execute(psutil.boot_time) - @unittest.skipIf(WINDOWS, "XXX produces a false positive on Windows") def test_users(self): self.execute(psutil.users) + def test_set_debug(self): + self.execute(lambda: psutil._set_debug(False)) + if WINDOWS: # --- win services @@ -589,17 +476,17 @@ class TestModuleFunctionsLeaks(TestMemLeak): def test_win_service_get_config(self): name = next(psutil.win_service_iter()).name() - self.execute(cext.winservice_query_config, name) + self.execute(lambda: cext.winservice_query_config(name)) def test_win_service_get_status(self): name = next(psutil.win_service_iter()).name() - self.execute(cext.winservice_query_status, name) + self.execute(lambda: cext.winservice_query_status(name)) def test_win_service_get_description(self): name = next(psutil.win_service_iter()).name() - self.execute(cext.winservice_query_descr, name) + self.execute(lambda: cext.winservice_query_descr(name)) if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 616a3654..70f6a37e 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. @@ -11,90 +11,76 @@ Miscellaneous tests. import ast import collections -import contextlib import errno import json import os import pickle import socket import stat +import sys -from psutil import FREEBSD from psutil import LINUX -from psutil import NETBSD from psutil import POSIX from psutil import WINDOWS +from psutil._common import debug from psutil._common import memoize from psutil._common import memoize_when_activated from psutil._common import supports_ipv6 from psutil._common import wrap_numbers -from psutil._common import open_text -from psutil._common import open_binary from psutil._compat import PY3 +from psutil._compat import redirect_stderr from psutil.tests import APPVEYOR -from psutil.tests import bind_socket -from psutil.tests import bind_unix_socket -from psutil.tests import call_until -from psutil.tests import chdir from psutil.tests import CI_TESTING -from psutil.tests import create_proc_children_pair -from psutil.tests import create_sockets -from psutil.tests import create_zombie_proc -from psutil.tests import DEVNULL -from psutil.tests import get_free_port -from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY -from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import HAS_NET_IO_COUNTERS from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import import_module_by_path -from psutil.tests import is_namedtuple from psutil.tests import mock +from psutil.tests import PsutilTestCase from psutil.tests import PYTHON_EXE -from psutil.tests import reap_children from psutil.tests import reload_module -from psutil.tests import retry from psutil.tests import ROOT_DIR -from psutil.tests import safe_mkdir -from psutil.tests import safe_rmpath from psutil.tests import SCRIPTS_DIR from psutil.tests import sh -from psutil.tests import tcp_socketpair -from psutil.tests import TESTFN -from psutil.tests import TOX -from psutil.tests import TRAVIS from psutil.tests import unittest -from psutil.tests import unix_socket_path -from psutil.tests import unix_socketpair -from psutil.tests import wait_for_file -from psutil.tests import wait_for_pid import psutil import psutil.tests +PYTHON_39 = sys.version_info[:2] == (3, 9) + + # =================================================================== # --- Misc / generic tests. # =================================================================== -class TestMisc(unittest.TestCase): +class TestMisc(PsutilTestCase): def test_process__repr__(self, func=repr): - p = psutil.Process() + p = psutil.Process(self.spawn_testproc().pid) r = func(p) self.assertIn("psutil.Process", r) self.assertIn("pid=%s" % p.pid, r) - self.assertIn("name=", r) - self.assertIn(p.name(), r) + self.assertIn("name='%s'" % str(p.name()), + r.replace("name=u'", "name='")) + self.assertIn("status=", r) + self.assertNotIn("exitcode=", r) + p.terminate() + p.wait() + r = func(p) + self.assertIn("status='terminated'", r) + self.assertIn("exitcode=", r) + with mock.patch.object(psutil.Process, "name", side_effect=psutil.ZombieProcess(os.getpid())): p = psutil.Process() r = func(p) self.assertIn("pid=%s" % p.pid, r) - self.assertIn("zombie", r) + self.assertIn("status='zombie'", r) self.assertNotIn("name=", r) with mock.patch.object(psutil.Process, "name", side_effect=psutil.NoSuchProcess(os.getpid())): @@ -113,57 +99,71 @@ class TestMisc(unittest.TestCase): def test_process__str__(self): self.test_process__repr__(func=str) - def test_no_such_process__repr__(self, func=repr): + def test_no_such_process__repr__(self): self.assertEqual( repr(psutil.NoSuchProcess(321)), - "psutil.NoSuchProcess process no longer exists (pid=321)") + "psutil.NoSuchProcess(pid=321, msg='process no longer exists')") self.assertEqual( - repr(psutil.NoSuchProcess(321, name='foo')), - "psutil.NoSuchProcess process no longer exists (pid=321, " - "name='foo')") + repr(psutil.NoSuchProcess(321, name="name", msg="msg")), + "psutil.NoSuchProcess(pid=321, name='name', msg='msg')") + + def test_no_such_process__str__(self): + self.assertEqual( + str(psutil.NoSuchProcess(321)), + "process no longer exists (pid=321)") self.assertEqual( - repr(psutil.NoSuchProcess(321, msg='foo')), - "psutil.NoSuchProcess foo") + str(psutil.NoSuchProcess(321, name="name", msg="msg")), + "msg (pid=321, name='name')") - def test_zombie_process__repr__(self, func=repr): + def test_zombie_process__repr__(self): self.assertEqual( repr(psutil.ZombieProcess(321)), - "psutil.ZombieProcess process still exists but it's a zombie " - "(pid=321)") + 'psutil.ZombieProcess(pid=321, msg="PID still ' + 'exists but it\'s a zombie")') self.assertEqual( - repr(psutil.ZombieProcess(321, name='foo')), - "psutil.ZombieProcess process still exists but it's a zombie " - "(pid=321, name='foo')") + repr(psutil.ZombieProcess(321, name="name", ppid=320, msg="foo")), + "psutil.ZombieProcess(pid=321, ppid=320, name='name', msg='foo')") + + def test_zombie_process__str__(self): self.assertEqual( - repr(psutil.ZombieProcess(321, name='foo', ppid=1)), - "psutil.ZombieProcess process still exists but it's a zombie " - "(pid=321, name='foo', ppid=1)") + str(psutil.ZombieProcess(321)), + "PID still exists but it's a zombie (pid=321)") self.assertEqual( - repr(psutil.ZombieProcess(321, msg='foo')), - "psutil.ZombieProcess foo") + str(psutil.ZombieProcess(321, name="name", ppid=320, msg="foo")), + "foo (pid=321, ppid=320, name='name')") - def test_access_denied__repr__(self, func=repr): + def test_access_denied__repr__(self): self.assertEqual( repr(psutil.AccessDenied(321)), - "psutil.AccessDenied (pid=321)") + "psutil.AccessDenied(pid=321)") self.assertEqual( - repr(psutil.AccessDenied(321, name='foo')), - "psutil.AccessDenied (pid=321, name='foo')") + repr(psutil.AccessDenied(321, name="name", msg="msg")), + "psutil.AccessDenied(pid=321, name='name', msg='msg')") + + def test_access_denied__str__(self): + self.assertEqual( + str(psutil.AccessDenied(321)), + "(pid=321)") self.assertEqual( - repr(psutil.AccessDenied(321, msg='foo')), - "psutil.AccessDenied foo") + str(psutil.AccessDenied(321, name="name", msg="msg")), + "msg (pid=321, name='name')") - def test_timeout_expired__repr__(self, func=repr): + def test_timeout_expired__repr__(self): + self.assertEqual( + repr(psutil.TimeoutExpired(5)), + "psutil.TimeoutExpired(seconds=5, msg='timeout after 5 seconds')") self.assertEqual( - repr(psutil.TimeoutExpired(321)), - "psutil.TimeoutExpired timeout after 321 seconds") + repr(psutil.TimeoutExpired(5, pid=321, name="name")), + "psutil.TimeoutExpired(pid=321, name='name', seconds=5, " + "msg='timeout after 5 seconds')") + + def test_timeout_expired__str__(self): self.assertEqual( - repr(psutil.TimeoutExpired(321, pid=111)), - "psutil.TimeoutExpired timeout after 321 seconds (pid=111)") + str(psutil.TimeoutExpired(5)), + "timeout after 5 seconds") self.assertEqual( - repr(psutil.TimeoutExpired(321, pid=111, name='foo')), - "psutil.TimeoutExpired timeout after 321 seconds " - "(pid=111, name='foo')") + str(psutil.TimeoutExpired(5, pid=321, name="name")), + "timeout after 5 seconds (pid=321, name='name')") def test_process__eq__(self): p1 = psutil.Process() @@ -180,9 +180,7 @@ class TestMisc(unittest.TestCase): def test__all__(self): dir_psutil = dir(psutil) for name in dir_psutil: - if name in ('callable', 'error', 'namedtuple', 'tests', - 'long', 'test', 'NUM_CPUS', 'BOOT_TIME', - 'TOTAL_PHYMEM', 'PermissionError', + if name in ('long', 'tests', 'test', 'PermissionError', 'ProcessLookupError'): continue if not name.startswith('_'): @@ -327,7 +325,10 @@ class TestMisc(unittest.TestCase): else: with self.assertRaises(Exception): sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - sock.bind(("::1", 0)) + try: + sock.bind(("::1", 0)) + finally: + sock.close() def test_isfile_strict(self): from psutil._common import isfile_strict @@ -341,7 +342,7 @@ class TestMisc(unittest.TestCase): side_effect=OSError(errno.EACCES, "foo")): self.assertRaises(OSError, isfile_strict, this_file) with mock.patch('psutil._common.os.stat', - side_effect=OSError(errno.EINVAL, "foo")): + side_effect=OSError(errno.ENOENT, "foo")): assert not isfile_strict(this_file) with mock.patch('psutil._common.stat.S_ISREG', return_value=False): assert not isfile_strict(this_file) @@ -371,7 +372,7 @@ class TestMisc(unittest.TestCase): def test_setup_script(self): setup_py = os.path.join(ROOT_DIR, 'setup.py') - if TRAVIS and not os.path.exists(setup_py): + if CI_TESTING and not os.path.exists(setup_py): return self.skipTest("can't find setup.py") module = import_module_by_path(setup_py) self.assertRaises(SystemExit, module.setup) @@ -402,6 +403,35 @@ class TestMisc(unittest.TestCase): reload_module(psutil) self.assertIn("version conflict", str(cm.exception).lower()) + def test_debug(self): + if PY3: + from io import StringIO + else: + from StringIO import StringIO + + with redirect_stderr(StringIO()) as f: + debug("hello") + msg = f.getvalue() + assert msg.startswith("psutil-debug"), msg + self.assertIn("hello", msg) + self.assertIn(__file__, msg) + + # supposed to use repr(exc) + with redirect_stderr(StringIO()) as f: + debug(ValueError("this is an error")) + msg = f.getvalue() + self.assertIn("ignoring ValueError", msg) + self.assertIn("'this is an error'", msg) + + # supposed to use str(exc), because of extra info about file name + with redirect_stderr(StringIO()) as f: + exc = OSError(2, "no such file") + exc.filename = "/foo" + debug(exc) + msg = f.getvalue() + self.assertIn("no such file", msg) + self.assertIn("/foo", msg) + # =================================================================== # --- Tests for wrap_numbers() function. @@ -411,7 +441,7 @@ class TestMisc(unittest.TestCase): nt = collections.namedtuple('foo', 'a b c') -class TestWrapNumbers(unittest.TestCase): +class TestWrapNumbers(PsutilTestCase): def setUp(self): wrap_numbers.cache_clear() @@ -650,11 +680,9 @@ class TestWrapNumbers(unittest.TestCase): # =================================================================== -@unittest.skipIf(TOX, "can't test on TOX") -# See: https://travis-ci.org/giampaolo/psutil/jobs/295224806 -@unittest.skipIf(TRAVIS and not os.path.exists(SCRIPTS_DIR), +@unittest.skipIf(not os.path.exists(SCRIPTS_DIR), "can't locate scripts directory") -class TestScripts(unittest.TestCase): +class TestScripts(PsutilTestCase): """Tests for scripts in the "scripts" directory.""" @staticmethod @@ -727,8 +755,6 @@ class TestScripts(unittest.TestCase): def test_netstat(self): self.assert_stdout('netstat.py') - # permission denied on travis - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_ifconfig(self): self.assert_stdout('ifconfig.py') @@ -739,7 +765,7 @@ class TestScripts(unittest.TestCase): def test_procsmem(self): if 'uss' not in psutil.Process().memory_full_info()._fields: raise self.skipTest("not supported") - self.assert_stdout('procsmem.py', stderr=DEVNULL) + self.assert_stdout('procsmem.py') def test_killall(self): self.assert_syntax('killall.py') @@ -765,14 +791,12 @@ class TestScripts(unittest.TestCase): self.assert_syntax('cpu_distribution.py') @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_temperatures(self): if not psutil.sensors_temperatures(): self.skipTest("no temperatures") self.assert_stdout('temperatures.py') @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_fans(self): if not psutil.sensors_fans(): self.skipTest("no fans") @@ -783,283 +807,12 @@ class TestScripts(unittest.TestCase): def test_battery(self): self.assert_stdout('battery.py') + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_sensors(self): self.assert_stdout('sensors.py') -# =================================================================== -# --- Unit tests for test utilities. -# =================================================================== - - -class TestRetryDecorator(unittest.TestCase): - - @mock.patch('time.sleep') - def test_retry_success(self, sleep): - # Fail 3 times out of 5; make sure the decorated fun returns. - - @retry(retries=5, interval=1, logfun=None) - def foo(): - while queue: - queue.pop() - 1 / 0 - return 1 - - queue = list(range(3)) - self.assertEqual(foo(), 1) - self.assertEqual(sleep.call_count, 3) - - @mock.patch('time.sleep') - def test_retry_failure(self, sleep): - # Fail 6 times out of 5; th function is supposed to raise exc. - - @retry(retries=5, interval=1, logfun=None) - def foo(): - while queue: - queue.pop() - 1 / 0 - return 1 - - queue = list(range(6)) - self.assertRaises(ZeroDivisionError, foo) - self.assertEqual(sleep.call_count, 5) - - @mock.patch('time.sleep') - def test_exception_arg(self, sleep): - @retry(exception=ValueError, interval=1) - def foo(): - raise TypeError - - self.assertRaises(TypeError, foo) - self.assertEqual(sleep.call_count, 0) - - @mock.patch('time.sleep') - def test_no_interval_arg(self, sleep): - # if interval is not specified sleep is not supposed to be called - - @retry(retries=5, interval=None, logfun=None) - def foo(): - 1 / 0 - - self.assertRaises(ZeroDivisionError, foo) - self.assertEqual(sleep.call_count, 0) - - @mock.patch('time.sleep') - def test_retries_arg(self, sleep): - - @retry(retries=5, interval=1, logfun=None) - def foo(): - 1 / 0 - - self.assertRaises(ZeroDivisionError, foo) - self.assertEqual(sleep.call_count, 5) - - @mock.patch('time.sleep') - def test_retries_and_timeout_args(self, sleep): - self.assertRaises(ValueError, retry, retries=5, timeout=1) - - -class TestSyncTestUtils(unittest.TestCase): - - def tearDown(self): - safe_rmpath(TESTFN) - - def test_wait_for_pid(self): - wait_for_pid(os.getpid()) - nopid = max(psutil.pids()) + 99999 - with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): - self.assertRaises(psutil.NoSuchProcess, wait_for_pid, nopid) - - def test_wait_for_file(self): - with open(TESTFN, 'w') as f: - f.write('foo') - wait_for_file(TESTFN) - assert not os.path.exists(TESTFN) - - def test_wait_for_file_empty(self): - with open(TESTFN, 'w'): - pass - wait_for_file(TESTFN, empty=True) - assert not os.path.exists(TESTFN) - - def test_wait_for_file_no_file(self): - with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): - self.assertRaises(IOError, wait_for_file, TESTFN) - - def test_wait_for_file_no_delete(self): - with open(TESTFN, 'w') as f: - f.write('foo') - wait_for_file(TESTFN, delete=False) - assert os.path.exists(TESTFN) - - def test_call_until(self): - ret = call_until(lambda: 1, "ret == 1") - self.assertEqual(ret, 1) - - -class TestFSTestUtils(unittest.TestCase): - - def setUp(self): - safe_rmpath(TESTFN) - - tearDown = setUp - - def test_open_text(self): - with open_text(__file__) as f: - self.assertEqual(f.mode, 'rt') - - def test_open_binary(self): - with open_binary(__file__) as f: - self.assertEqual(f.mode, 'rb') - - def test_safe_mkdir(self): - safe_mkdir(TESTFN) - assert os.path.isdir(TESTFN) - safe_mkdir(TESTFN) - assert os.path.isdir(TESTFN) - - def test_safe_rmpath(self): - # test file is removed - open(TESTFN, 'w').close() - safe_rmpath(TESTFN) - assert not os.path.exists(TESTFN) - # test no exception if path does not exist - safe_rmpath(TESTFN) - # test dir is removed - os.mkdir(TESTFN) - safe_rmpath(TESTFN) - assert not os.path.exists(TESTFN) - # test other exceptions are raised - with mock.patch('psutil.tests.os.stat', - side_effect=OSError(errno.EINVAL, "")) as m: - with self.assertRaises(OSError): - safe_rmpath(TESTFN) - assert m.called - - def test_chdir(self): - base = os.getcwd() - os.mkdir(TESTFN) - with chdir(TESTFN): - self.assertEqual(os.getcwd(), os.path.join(base, TESTFN)) - self.assertEqual(os.getcwd(), base) - - -class TestProcessUtils(unittest.TestCase): - - def test_reap_children(self): - subp = get_test_subprocess() - p = psutil.Process(subp.pid) - assert p.is_running() - reap_children() - assert not p.is_running() - assert not psutil.tests._pids_started - assert not psutil.tests._subprocesses_started - - def test_create_proc_children_pair(self): - p1, p2 = create_proc_children_pair() - self.assertNotEqual(p1.pid, p2.pid) - assert p1.is_running() - assert p2.is_running() - children = psutil.Process().children(recursive=True) - self.assertEqual(len(children), 2) - self.assertIn(p1, children) - self.assertIn(p2, children) - self.assertEqual(p1.ppid(), os.getpid()) - self.assertEqual(p2.ppid(), p1.pid) - - # make sure both of them are cleaned up - reap_children() - assert not p1.is_running() - assert not p2.is_running() - assert not psutil.tests._pids_started - assert not psutil.tests._subprocesses_started - - @unittest.skipIf(not POSIX, "POSIX only") - def test_create_zombie_proc(self): - zpid = create_zombie_proc() - self.addCleanup(reap_children, recursive=True) - p = psutil.Process(zpid) - self.assertEqual(p.status(), psutil.STATUS_ZOMBIE) - - -class TestNetUtils(unittest.TestCase): - - def bind_socket(self): - port = get_free_port() - with contextlib.closing(bind_socket(addr=('', port))) as s: - self.assertEqual(s.getsockname()[1], port) - - @unittest.skipIf(not POSIX, "POSIX only") - def test_bind_unix_socket(self): - with unix_socket_path() as name: - sock = bind_unix_socket(name) - with contextlib.closing(sock): - self.assertEqual(sock.family, socket.AF_UNIX) - self.assertEqual(sock.type, socket.SOCK_STREAM) - self.assertEqual(sock.getsockname(), name) - assert os.path.exists(name) - assert stat.S_ISSOCK(os.stat(name).st_mode) - # UDP - with unix_socket_path() as name: - sock = bind_unix_socket(name, type=socket.SOCK_DGRAM) - with contextlib.closing(sock): - self.assertEqual(sock.type, socket.SOCK_DGRAM) - - def tcp_tcp_socketpair(self): - addr = ("127.0.0.1", get_free_port()) - server, client = tcp_socketpair(socket.AF_INET, addr=addr) - with contextlib.closing(server): - with contextlib.closing(client): - # Ensure they are connected and the positions are - # correct. - self.assertEqual(server.getsockname(), addr) - self.assertEqual(client.getpeername(), addr) - self.assertNotEqual(client.getsockname(), addr) - - @unittest.skipIf(not POSIX, "POSIX only") - @unittest.skipIf(NETBSD or FREEBSD, - "/var/run/log UNIX socket opened by default") - def test_unix_socketpair(self): - p = psutil.Process() - num_fds = p.num_fds() - assert not p.connections(kind='unix') - with unix_socket_path() as name: - server, client = unix_socketpair(name) - try: - assert os.path.exists(name) - assert stat.S_ISSOCK(os.stat(name).st_mode) - self.assertEqual(p.num_fds() - num_fds, 2) - self.assertEqual(len(p.connections(kind='unix')), 2) - self.assertEqual(server.getsockname(), name) - self.assertEqual(client.getpeername(), name) - finally: - client.close() - server.close() - - def test_create_sockets(self): - with create_sockets() as socks: - fams = collections.defaultdict(int) - types = collections.defaultdict(int) - for s in socks: - fams[s.family] += 1 - # work around http://bugs.python.org/issue30204 - types[s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)] += 1 - self.assertGreaterEqual(fams[socket.AF_INET], 2) - if supports_ipv6(): - self.assertGreaterEqual(fams[socket.AF_INET6], 2) - if POSIX and HAS_CONNECTIONS_UNIX: - self.assertGreaterEqual(fams[socket.AF_UNIX], 2) - self.assertGreaterEqual(types[socket.SOCK_STREAM], 2) - self.assertGreaterEqual(types[socket.SOCK_DGRAM], 2) - - -class TestOtherUtils(unittest.TestCase): - - def test_is_namedtuple(self): - assert is_namedtuple(collections.namedtuple('foo', 'a b c')(1, 2, 3)) - assert not is_namedtuple(tuple()) - - if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 723b255e..f797de71 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -1,28 +1,29 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""MACOS specific tests.""" +"""macOS specific tests.""" -import os import re import time import psutil from psutil import MACOS -from psutil.tests import create_zombie_proc -from psutil.tests import get_test_subprocess +from psutil import POSIX from psutil.tests import HAS_BATTERY -from psutil.tests import MEMORY_TOLERANCE -from psutil.tests import reap_children +from psutil.tests import PsutilTestCase from psutil.tests import retry_on_failure from psutil.tests import sh +from psutil.tests import spawn_testproc +from psutil.tests import terminate +from psutil.tests import TOLERANCE_DISK_USAGE +from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import unittest - -PAGESIZE = os.sysconf("SC_PAGE_SIZE") if MACOS else None +if POSIX: + from psutil._psutil_posix import getpagesize def sysctl(cmdline): @@ -45,7 +46,7 @@ def vm_stat(field): break else: raise ValueError("line not found") - return int(re.search(r'\d+', line).group(0)) * PAGESIZE + return int(re.search(r'\d+', line).group(0)) * getpagesize() # http://code.activestate.com/recipes/578019/ @@ -76,15 +77,15 @@ def human2bytes(s): @unittest.skipIf(not MACOS, "MACOS only") -class TestProcess(unittest.TestCase): +class TestProcess(PsutilTestCase): @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess().pid + cls.pid = spawn_testproc().pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) def test_process_create_time(self): output = sh("ps -o lstart -p %s" % self.pid) @@ -101,68 +102,11 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(not MACOS, "MACOS only") -class TestZombieProcessAPIs(unittest.TestCase): - - @classmethod - def setUpClass(cls): - zpid = create_zombie_proc() - cls.p = psutil.Process(zpid) - - @classmethod - def tearDownClass(cls): - reap_children(recursive=True) - - def test_pidtask_info(self): - self.assertEqual(self.p.status(), psutil.STATUS_ZOMBIE) - self.p.ppid() - self.p.uids() - self.p.gids() - self.p.terminal() - self.p.create_time() - - def test_exe(self): - self.assertRaises(psutil.ZombieProcess, self.p.exe) - - def test_cmdline(self): - self.assertRaises(psutil.ZombieProcess, self.p.cmdline) - - def test_environ(self): - self.assertRaises(psutil.ZombieProcess, self.p.environ) - - def test_cwd(self): - self.assertRaises(psutil.ZombieProcess, self.p.cwd) - - def test_memory_full_info(self): - self.assertRaises(psutil.ZombieProcess, self.p.memory_full_info) - - def test_cpu_times(self): - self.assertRaises(psutil.ZombieProcess, self.p.cpu_times) - - def test_num_ctx_switches(self): - self.assertRaises(psutil.ZombieProcess, self.p.num_ctx_switches) - - def test_num_threads(self): - self.assertRaises(psutil.ZombieProcess, self.p.num_threads) - - def test_open_files(self): - self.assertRaises(psutil.ZombieProcess, self.p.open_files) - - def test_connections(self): - self.assertRaises(psutil.ZombieProcess, self.p.connections) - - def test_num_fds(self): - self.assertRaises(psutil.ZombieProcess, self.p.num_fds) - - def test_threads(self): - self.assertRaises((psutil.ZombieProcess, psutil.AccessDenied), - self.p.threads) - - -@unittest.skipIf(not MACOS, "MACOS only") -class TestSystemAPIs(unittest.TestCase): +class TestSystemAPIs(PsutilTestCase): # --- disk + @retry_on_failure() def test_disks(self): # test psutil.disk_usage() and psutil.disk_partitions() # against "df -a" @@ -184,11 +128,10 @@ class TestSystemAPIs(unittest.TestCase): dev, total, used, free = df(part.mountpoint) self.assertEqual(part.device, dev) self.assertEqual(usage.total, total) - # 10 MB tollerance - if abs(usage.free - free) > 10 * 1024 * 1024: - self.fail("psutil=%s, df=%s" % usage.free, free) - if abs(usage.used - used) > 10 * 1024 * 1024: - self.fail("psutil=%s, df=%s" % usage.used, used) + self.assertAlmostEqual(usage.free, free, + delta=TOLERANCE_DISK_USAGE) + self.assertAlmostEqual(usage.used, used, + delta=TOLERANCE_DISK_USAGE) # --- cpu @@ -196,7 +139,7 @@ class TestSystemAPIs(unittest.TestCase): num = sysctl("sysctl hw.logicalcpu") self.assertEqual(num, psutil.cpu_count(logical=True)) - def test_cpu_count_physical(self): + def test_cpu_count_cores(self): num = sysctl("sysctl hw.physicalcpu") self.assertEqual(num, psutil.cpu_count(logical=False)) @@ -219,25 +162,25 @@ class TestSystemAPIs(unittest.TestCase): def test_vmem_free(self): vmstat_val = vm_stat("free") psutil_val = psutil.virtual_memory().free - self.assertAlmostEqual(psutil_val, vmstat_val, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_active(self): vmstat_val = vm_stat("active") psutil_val = psutil.virtual_memory().active - self.assertAlmostEqual(psutil_val, vmstat_val, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_inactive(self): vmstat_val = vm_stat("inactive") psutil_val = psutil.virtual_memory().inactive - self.assertAlmostEqual(psutil_val, vmstat_val, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_vmem_wired(self): vmstat_val = vm_stat("wired") psutil_val = psutil.virtual_memory().wired - self.assertAlmostEqual(psutil_val, vmstat_val, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) # --- swap mem @@ -290,5 +233,5 @@ class TestSystemAPIs(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index e405393b..acb6aa20 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. @@ -23,20 +23,24 @@ from psutil import OPENBSD from psutil import POSIX from psutil import SUNOS from psutil.tests import CI_TESTING -from psutil.tests import get_kernel_version -from psutil.tests import get_test_subprocess +from psutil.tests import spawn_testproc from psutil.tests import HAS_NET_IO_COUNTERS from psutil.tests import mock +from psutil.tests import PsutilTestCase from psutil.tests import PYTHON_EXE -from psutil.tests import reap_children from psutil.tests import retry_on_failure from psutil.tests import sh from psutil.tests import skip_on_access_denied -from psutil.tests import TRAVIS +from psutil.tests import terminate from psutil.tests import unittest -from psutil.tests import wait_for_pid from psutil.tests import which +if POSIX: + import mmap + import resource + + from psutil._psutil_posix import getpagesize + def ps(fmt, pid=None): """ @@ -58,8 +62,7 @@ def ps(fmt, pid=None): cmd.append('ax') if SUNOS: - fmt_map = {'command', 'comm', - 'start', 'stime'} + fmt_map = set(('command', 'comm', 'start', 'stime')) fmt = fmt_map.get(fmt, fmt) cmd.extend(['-o', fmt]) @@ -128,18 +131,17 @@ def ps_vsz(pid): @unittest.skipIf(not POSIX, "POSIX only") -class TestProcess(unittest.TestCase): +class TestProcess(PsutilTestCase): """Compare psutil results against 'ps' command line utility (mainly).""" @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess([PYTHON_EXE, "-E", "-O"], - stdin=subprocess.PIPE).pid - wait_for_pid(cls.pid) + cls.pid = spawn_testproc([PYTHON_EXE, "-E", "-O"], + stdin=subprocess.PIPE).pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) def test_ppid(self): ppid_ps = ps('ppid', self.pid) @@ -285,50 +287,9 @@ class TestProcess(unittest.TestCase): psutil_nice = psutil.Process().nice() self.assertEqual(ps_nice, psutil_nice) - def test_num_fds(self): - # Note: this fails from time to time; I'm keen on thinking - # it doesn't mean something is broken - def call(p, attr): - args = () - attr = getattr(p, name, None) - if attr is not None and callable(attr): - if name == 'rlimit': - args = (psutil.RLIMIT_NOFILE,) - attr(*args) - else: - attr - - p = psutil.Process(os.getpid()) - failures = [] - ignored_names = ['terminate', 'kill', 'suspend', 'resume', 'nice', - 'send_signal', 'wait', 'children', 'as_dict', - 'memory_info_ex', 'parent', 'parents'] - if LINUX and get_kernel_version() < (2, 6, 36): - ignored_names.append('rlimit') - if LINUX and get_kernel_version() < (2, 6, 23): - ignored_names.append('num_ctx_switches') - for name in dir(psutil.Process): - if (name.startswith('_') or name in ignored_names): - continue - else: - try: - num1 = p.num_fds() - for x in range(2): - call(p, name) - num2 = p.num_fds() - except psutil.AccessDenied: - pass - else: - if abs(num2 - num1) > 1: - fail = "failure while processing Process.%s method " \ - "(before=%s, after=%s)" % (name, num1, num2) - failures.append(fail) - if failures: - self.fail('\n' + '\n'.join(failures)) - @unittest.skipIf(not POSIX, "POSIX only") -class TestSystemAPIs(unittest.TestCase): +class TestSystemAPIs(PsutilTestCase): """Test some system APIs.""" @retry_on_failure() @@ -351,7 +312,6 @@ class TestSystemAPIs(unittest.TestCase): # for some reason ifconfig -a does not report all interfaces # returned by psutil @unittest.skipIf(SUNOS, "unreliable on SUNOS") - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @unittest.skipIf(not which('ifconfig'), "no ifconfig cmd") @unittest.skipIf(not HAS_NET_IO_COUNTERS, "not supported") def test_nic_names(self): @@ -369,9 +329,9 @@ class TestSystemAPIs(unittest.TestCase): @retry_on_failure() def test_users(self): out = sh("who") - lines = out.split('\n') - if not lines: + if not out.strip(): raise self.skipTest("no users on this system") + lines = out.split('\n') users = [x.split()[0] for x in lines] terminals = [x.split()[1] for x in lines] self.assertEqual(len(users), len(psutil.users())) @@ -415,6 +375,7 @@ class TestSystemAPIs(unittest.TestCase): # AIX can return '-' in df output instead of numbers, e.g. for /proc @unittest.skipIf(AIX, "unreliable on AIX") + @retry_on_failure() def test_disk_usage(self): def df(device): out = sh("df -k %s" % device).strip() @@ -449,6 +410,16 @@ class TestSystemAPIs(unittest.TestCase): self.assertAlmostEqual(usage.percent, percent, delta=1) +@unittest.skipIf(not POSIX, "POSIX only") +class TestMisc(PsutilTestCase): + + def test_getpagesize(self): + pagesize = getpagesize() + self.assertGreater(pagesize, 0) + self.assertEqual(pagesize, resource.getpagesize()) + self.assertEqual(pagesize, mmap.PAGESIZE) + + if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 0277a56a..fe74e601 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -13,9 +13,9 @@ import itertools import os import signal import socket +import stat import subprocess import sys -import tempfile import textwrap import time import types @@ -33,17 +33,17 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS from psutil._common import open_text +from psutil._compat import FileNotFoundError from psutil._compat import long from psutil._compat import PY3 +from psutil._compat import super from psutil.tests import APPVEYOR from psutil.tests import call_until -from psutil.tests import CIRRUS +from psutil.tests import CI_TESTING from psutil.tests import copyload_shared_lib from psutil.tests import create_exe -from psutil.tests import create_proc_children_pair -from psutil.tests import create_zombie_proc -from psutil.tests import enum -from psutil.tests import get_test_subprocess +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import GLOBAL_TIMEOUT from psutil.tests import HAS_CPU_AFFINITY from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE @@ -53,18 +53,16 @@ from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_THREADS from psutil.tests import mock +from psutil.tests import process_namespace +from psutil.tests import PsutilTestCase from psutil.tests import PYPY from psutil.tests import PYTHON_EXE from psutil.tests import reap_children from psutil.tests import retry_on_failure -from psutil.tests import safe_rmpath from psutil.tests import sh from psutil.tests import skip_on_access_denied from psutil.tests import skip_on_not_implemented -from psutil.tests import TESTFILE_PREFIX -from psutil.tests import TESTFN from psutil.tests import ThreadTask -from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import wait_for_pid @@ -73,162 +71,160 @@ from psutil.tests import wait_for_pid # --- psutil.Process class tests # =================================================================== -class TestProcess(unittest.TestCase): + +class TestProcess(PsutilTestCase): """Tests for psutil.Process class.""" - def setUp(self): - safe_rmpath(TESTFN) + def spawn_psproc(self, *args, **kwargs): + sproc = self.spawn_testproc(*args, **kwargs) + return psutil.Process(sproc.pid) - def tearDown(self): - reap_children() + # --- def test_pid(self): p = psutil.Process() self.assertEqual(p.pid, os.getpid()) - sproc = get_test_subprocess() - self.assertEqual(psutil.Process(sproc.pid).pid, sproc.pid) with self.assertRaises(AttributeError): p.pid = 33 def test_kill(self): - sproc = get_test_subprocess() - test_pid = sproc.pid - p = psutil.Process(test_pid) + p = self.spawn_psproc() p.kill() - sig = p.wait() - self.assertFalse(psutil.pid_exists(test_pid)) - if POSIX: - self.assertEqual(sig, -signal.SIGKILL) + code = p.wait() + if WINDOWS: + self.assertEqual(code, signal.SIGTERM) + else: + self.assertEqual(code, -signal.SIGKILL) + self.assertProcessGone(p) def test_terminate(self): - sproc = get_test_subprocess() - test_pid = sproc.pid - p = psutil.Process(test_pid) + p = self.spawn_psproc() p.terminate() - sig = p.wait() - self.assertFalse(psutil.pid_exists(test_pid)) - if POSIX: - self.assertEqual(sig, -signal.SIGTERM) + code = p.wait() + if WINDOWS: + self.assertEqual(code, signal.SIGTERM) + else: + self.assertEqual(code, -signal.SIGTERM) + self.assertProcessGone(p) def test_send_signal(self): sig = signal.SIGKILL if POSIX else signal.SIGTERM - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() p.send_signal(sig) - exit_sig = p.wait() - self.assertFalse(psutil.pid_exists(p.pid)) - if POSIX: - self.assertEqual(exit_sig, -sig) - # - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - p.send_signal(sig) - with mock.patch('psutil.os.kill', - side_effect=OSError(errno.ESRCH, "")): - with self.assertRaises(psutil.NoSuchProcess): - p.send_signal(sig) - # - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - p.send_signal(sig) - with mock.patch('psutil.os.kill', - side_effect=OSError(errno.EPERM, "")): - with self.assertRaises(psutil.AccessDenied): - psutil.Process().send_signal(sig) - # Sending a signal to process with PID 0 is not allowed as - # it would affect every process in the process group of - # the calling process (os.getpid()) instead of PID 0"). - if 0 in psutil.pids(): - p = psutil.Process(0) - self.assertRaises(ValueError, p.send_signal, signal.SIGTERM) - - def test_wait(self): - # check exit code signal - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - p.kill() code = p.wait() - if POSIX: - self.assertEqual(code, -signal.SIGKILL) + if WINDOWS: + self.assertEqual(code, sig) else: - self.assertEqual(code, signal.SIGTERM) - self.assertFalse(p.is_running()) - - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - p.terminate() + self.assertEqual(code, -sig) + self.assertProcessGone(p) + + @unittest.skipIf(not POSIX, "not POSIX") + def test_send_signal_mocked(self): + sig = signal.SIGTERM + p = self.spawn_psproc() + with mock.patch('psutil.os.kill', + side_effect=OSError(errno.ESRCH, "")): + self.assertRaises(psutil.NoSuchProcess, p.send_signal, sig) + + p = self.spawn_psproc() + with mock.patch('psutil.os.kill', + side_effect=OSError(errno.EPERM, "")): + self.assertRaises(psutil.AccessDenied, p.send_signal, sig) + + def test_wait_exited(self): + # Test waitpid() + WIFEXITED -> WEXITSTATUS. + # normal return, same as exit(0) + cmd = [PYTHON_EXE, "-c", "pass"] + p = self.spawn_psproc(cmd) + code = p.wait() + self.assertEqual(code, 0) + self.assertProcessGone(p) + # exit(1), implicit in case of error + cmd = [PYTHON_EXE, "-c", "1 / 0"] + p = self.spawn_psproc(cmd, stderr=subprocess.PIPE) + code = p.wait() + self.assertEqual(code, 1) + self.assertProcessGone(p) + # via sys.exit() + cmd = [PYTHON_EXE, "-c", "import sys; sys.exit(5);"] + p = self.spawn_psproc(cmd) code = p.wait() + self.assertEqual(code, 5) + self.assertProcessGone(p) + # via os._exit() + cmd = [PYTHON_EXE, "-c", "import os; os._exit(5);"] + p = self.spawn_psproc(cmd) + code = p.wait() + self.assertEqual(code, 5) + self.assertProcessGone(p) + + def test_wait_stopped(self): + p = self.spawn_psproc() if POSIX: - self.assertEqual(code, -signal.SIGTERM) + # Test waitpid() + WIFSTOPPED and WIFCONTINUED. + # Note: if a process is stopped it ignores SIGTERM. + p.send_signal(signal.SIGSTOP) + self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001) + p.send_signal(signal.SIGCONT) + self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001) + p.send_signal(signal.SIGTERM) + self.assertEqual(p.wait(), -signal.SIGTERM) + self.assertEqual(p.wait(), -signal.SIGTERM) else: - self.assertEqual(code, signal.SIGTERM) - self.assertFalse(p.is_running()) - - # check sys.exit() code - code = "import time, sys; time.sleep(0.01); sys.exit(5);" - sproc = get_test_subprocess([PYTHON_EXE, "-c", code]) - p = psutil.Process(sproc.pid) - self.assertEqual(p.wait(), 5) - self.assertFalse(p.is_running()) - - # Test wait() issued twice. - # It is not supposed to raise NSP when the process is gone. - # On UNIX this should return None, on Windows it should keep - # returning the exit code. - sproc = get_test_subprocess([PYTHON_EXE, "-c", code]) - p = psutil.Process(sproc.pid) - self.assertEqual(p.wait(), 5) - self.assertIn(p.wait(), (5, None)) - - # test timeout - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - p.name() - self.assertRaises(psutil.TimeoutExpired, p.wait, 0.01) - - # timeout < 0 not allowed - self.assertRaises(ValueError, p.wait, -1) + p.suspend() + self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001) + p.resume() + self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001) + p.terminate() + self.assertEqual(p.wait(), signal.SIGTERM) + self.assertEqual(p.wait(), signal.SIGTERM) def test_wait_non_children(self): # Test wait() against a process which is not our direct # child. - p1, p2 = create_proc_children_pair() - self.assertRaises(psutil.TimeoutExpired, p1.wait, 0.01) - self.assertRaises(psutil.TimeoutExpired, p2.wait, 0.01) + child, grandchild = self.spawn_children_pair() + self.assertRaises(psutil.TimeoutExpired, child.wait, 0.01) + self.assertRaises(psutil.TimeoutExpired, grandchild.wait, 0.01) # We also terminate the direct child otherwise the # grandchild will hang until the parent is gone. - p1.terminate() - p2.terminate() - ret1 = p1.wait() - ret2 = p2.wait() + child.terminate() + grandchild.terminate() + child_ret = child.wait() + grandchild_ret = grandchild.wait() if POSIX: - self.assertEqual(ret1, -signal.SIGTERM) + self.assertEqual(child_ret, -signal.SIGTERM) # For processes which are not our children we're supposed # to get None. - self.assertEqual(ret2, None) + self.assertEqual(grandchild_ret, None) else: - self.assertEqual(ret1, signal.SIGTERM) - self.assertEqual(ret1, signal.SIGTERM) + self.assertEqual(child_ret, signal.SIGTERM) + self.assertEqual(child_ret, signal.SIGTERM) - def test_wait_timeout_0(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + def test_wait_timeout(self): + p = self.spawn_psproc() + p.name() + self.assertRaises(psutil.TimeoutExpired, p.wait, 0.01) + self.assertRaises(psutil.TimeoutExpired, p.wait, 0) + self.assertRaises(ValueError, p.wait, -1) + + def test_wait_timeout_nonblocking(self): + p = self.spawn_psproc() self.assertRaises(psutil.TimeoutExpired, p.wait, 0) p.kill() - stop_at = time.time() + 2 - while True: + stop_at = time.time() + GLOBAL_TIMEOUT + while time.time() < stop_at: try: code = p.wait(0) - except psutil.TimeoutExpired: - if time.time() >= stop_at: - raise - else: break + except psutil.TimeoutExpired: + pass + else: + raise self.fail('timeout') if POSIX: self.assertEqual(code, -signal.SIGKILL) else: self.assertEqual(code, signal.SIGTERM) - self.assertFalse(p.is_running()) + self.assertProcessGone(p) def test_cpu_percent(self): p = psutil.Process() @@ -281,9 +277,8 @@ class TestProcess(unittest.TestCase): self.assertIn(p.cpu_num(), range(psutil.cpu_count())) def test_create_time(self): - sproc = get_test_subprocess() + p = self.spawn_psproc() now = time.time() - p = psutil.Process(sproc.pid) create_time = p.create_time() # Use time.time() as base value to compare our result using a @@ -298,20 +293,16 @@ class TestProcess(unittest.TestCase): time.strftime("%Y %m %d %H:%M:%S", time.localtime(p.create_time())) @unittest.skipIf(not POSIX, 'POSIX only') - @unittest.skipIf(TRAVIS or CIRRUS, 'not reliable on TRAVIS/CIRRUS') def test_terminal(self): terminal = psutil.Process().terminal() - if sys.stdin.isatty() or sys.stdout.isatty(): + if terminal is not None: tty = os.path.realpath(sh('tty')) self.assertEqual(terminal, tty) - else: - self.assertIsNone(terminal) @unittest.skipIf(not HAS_PROC_IO_COUNTERS, 'not supported') @skip_on_not_implemented(only_if=LINUX) def test_io_counters(self): p = psutil.Process() - # test reads io1 = p.io_counters() with open(PYTHON_EXE, 'rb') as f: @@ -329,7 +320,7 @@ class TestProcess(unittest.TestCase): # test writes io1 = p.io_counters() - with tempfile.TemporaryFile(prefix=TESTFILE_PREFIX) as f: + with open(self.get_testfn(), 'wb') as f: if PY3: f.write(bytes("x" * 1000000, 'ascii')) else: @@ -355,11 +346,13 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(not LINUX, "linux only") def test_ionice_linux(self): p = psutil.Process() - self.assertEqual(p.ionice()[0], psutil.IOPRIO_CLASS_NONE) + if not CI_TESTING: + self.assertEqual(p.ionice()[0], psutil.IOPRIO_CLASS_NONE) self.assertEqual(psutil.IOPRIO_CLASS_NONE, 0) self.assertEqual(psutil.IOPRIO_CLASS_RT, 1) # high self.assertEqual(psutil.IOPRIO_CLASS_BE, 2) # normal self.assertEqual(psutil.IOPRIO_CLASS_IDLE, 3) # low + init = p.ionice() try: # low p.ionice(psutil.IOPRIO_CLASS_IDLE) @@ -373,16 +366,10 @@ class TestProcess(unittest.TestCase): self.assertEqual(tuple(p.ionice()), (psutil.IOPRIO_CLASS_BE, 7)) with self.assertRaises(ValueError): p.ionice(psutil.IOPRIO_CLASS_BE, value=8) - # high - if os.getuid() == 0: # root - p.ionice(psutil.IOPRIO_CLASS_RT) - self.assertEqual(tuple(p.ionice()), - (psutil.IOPRIO_CLASS_RT, 0)) + try: p.ionice(psutil.IOPRIO_CLASS_RT, value=7) - self.assertEqual(tuple(p.ionice()), - (psutil.IOPRIO_CLASS_RT, 7)) - with self.assertRaises(ValueError): - p.ionice(psutil.IOPRIO_CLASS_IDLE, value=8) + except psutil.AccessDenied: + pass # errs self.assertRaisesRegex( ValueError, "ioclass accepts no value", @@ -394,13 +381,18 @@ class TestProcess(unittest.TestCase): ValueError, "'ioclass' argument must be specified", p.ionice, value=1) finally: - p.ionice(psutil.IOPRIO_CLASS_BE) + ioclass, value = init + if ioclass == psutil.IOPRIO_CLASS_NONE: + value = 0 + p.ionice(ioclass, value) @unittest.skipIf(not HAS_IONICE, "not supported") @unittest.skipIf(not WINDOWS, 'not supported on this win version') def test_ionice_win(self): p = psutil.Process() - self.assertEqual(p.ionice(), psutil.IOPRIO_NORMAL) + if not CI_TESTING: + self.assertEqual(p.ionice(), psutil.IOPRIO_NORMAL) + init = p.ionice() try: # base p.ionice(psutil.IOPRIO_VERYLOW) @@ -421,8 +413,7 @@ class TestProcess(unittest.TestCase): ValueError, "is not a valid priority", p.ionice, psutil.IOPRIO_HIGH + 1) finally: - p.ionice(psutil.IOPRIO_NORMAL) - self.assertEqual(p.ionice(), psutil.IOPRIO_NORMAL) + p.ionice(init) @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_get(self): @@ -449,29 +440,30 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_set(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() p.rlimit(psutil.RLIMIT_NOFILE, (5, 5)) self.assertEqual(p.rlimit(psutil.RLIMIT_NOFILE), (5, 5)) # If pid is 0 prlimit() applies to the calling process and # we don't want that. - with self.assertRaises(ValueError): - psutil._psplatform.Process(0).rlimit(0) + if LINUX: + with self.assertRaisesRegex(ValueError, "can't use prlimit"): + psutil._psplatform.Process(0).rlimit(0) with self.assertRaises(ValueError): p.rlimit(psutil.RLIMIT_NOFILE, (5, 5, 5)) @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit(self): p = psutil.Process() + testfn = self.get_testfn() soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) try: p.rlimit(psutil.RLIMIT_FSIZE, (1024, hard)) - with open(TESTFN, "wb") as f: + with open(testfn, "wb") as f: f.write(b"X" * 1024) # write() or flush() doesn't always cause the exception # but close() will. with self.assertRaises(IOError) as exc: - with open(TESTFN, "wb") as f: + with open(testfn, "wb") as f: f.write(b"X" * 1025) self.assertEqual(exc.exception.errno if PY3 else exc.exception[0], errno.EFBIG) @@ -488,7 +480,7 @@ class TestProcess(unittest.TestCase): try: p.rlimit(psutil.RLIMIT_FSIZE, (1024, hard)) p.rlimit(psutil.RLIMIT_FSIZE, (psutil.RLIM_INFINITY, hard)) - with open(TESTFN, "wb") as f: + with open(self.get_testfn(), "wb") as f: f.write(b"X" * 2048) finally: p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) @@ -553,8 +545,7 @@ class TestProcess(unittest.TestCase): @skip_on_access_denied(only_if=MACOS) @unittest.skipIf(not HAS_THREADS, 'not supported') def test_threads_2(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() if OPENBSD: try: p.threads() @@ -568,6 +559,7 @@ class TestProcess(unittest.TestCase): p.cpu_times().system, sum([x.system_time for x in p.threads()]), delta=0.1) + @retry_on_failure() def test_memory_info(self): p = psutil.Process() @@ -599,8 +591,9 @@ class TestProcess(unittest.TestCase): self.assertGreaterEqual(getattr(mem, name), 0) def test_memory_full_info(self): + p = psutil.Process() total = psutil.virtual_memory().total - mem = psutil.Process().memory_full_info() + mem = p.memory_full_info() for name in mem._fields: value = getattr(mem, name) self.assertGreaterEqual(value, 0, msg=(name, value)) @@ -642,7 +635,12 @@ class TestProcess(unittest.TestCase): # 64 bit dlls: they are visible via explorer but cannot # be accessed via os.stat() (wtf?). if '64' not in os.path.basename(nt.path): - assert os.path.exists(nt.path), nt.path + try: + st = os.stat(nt.path) + except FileNotFoundError: + pass + else: + assert stat.S_ISREG(st.st_mode), nt.path for nt in ext_maps: for fname in nt._fields: value = getattr(nt, fname) @@ -657,11 +655,12 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps_lists_lib(self): # Make sure a newly loaded shared lib is listed. + p = psutil.Process() with copyload_shared_lib() as path: def normpath(p): return os.path.realpath(os.path.normcase(p)) libpaths = [normpath(x.path) - for x in psutil.Process().memory_maps()] + for x in p.memory_maps()] self.assertIn(normpath(path), libpaths) def test_memory_percent(self): @@ -672,8 +671,7 @@ class TestProcess(unittest.TestCase): p.memory_percent(memtype='uss') def test_is_running(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() assert p.is_running() assert p.is_running() p.kill() @@ -682,8 +680,8 @@ class TestProcess(unittest.TestCase): assert not p.is_running() def test_exe(self): - sproc = get_test_subprocess() - exe = psutil.Process(sproc.pid).exe() + p = self.spawn_psproc() + exe = p.exe() try: self.assertEqual(exe, PYTHON_EXE) except AssertionError: @@ -711,45 +709,43 @@ class TestProcess(unittest.TestCase): def test_cmdline(self): cmdline = [PYTHON_EXE, "-c", "import time; time.sleep(60)"] - sproc = get_test_subprocess(cmdline) - try: - self.assertEqual(' '.join(psutil.Process(sproc.pid).cmdline()), - ' '.join(cmdline)) - except AssertionError: - # XXX - most of the times the underlying sysctl() call on Net - # and Open BSD returns a truncated string. - # Also /proc/pid/cmdline behaves the same so it looks - # like this is a kernel bug. - # XXX - AIX truncates long arguments in /proc/pid/cmdline - if NETBSD or OPENBSD or AIX: - self.assertEqual( - psutil.Process(sproc.pid).cmdline()[0], PYTHON_EXE) - else: - raise + p = self.spawn_psproc(cmdline) + # XXX - most of the times the underlying sysctl() call on Net + # and Open BSD returns a truncated string. + # Also /proc/pid/cmdline behaves the same so it looks + # like this is a kernel bug. + # XXX - AIX truncates long arguments in /proc/pid/cmdline + if NETBSD or OPENBSD or AIX: + self.assertEqual(p.cmdline()[0], PYTHON_EXE) + else: + if MACOS and CI_TESTING: + pyexe = p.cmdline()[0] + if pyexe != PYTHON_EXE: + self.assertEqual(' '.join(p.cmdline()[1:]), + ' '.join(cmdline[1:])) + return + self.assertEqual(' '.join(p.cmdline()), ' '.join(cmdline)) @unittest.skipIf(PYPY, "broken on PYPY") def test_long_cmdline(self): - create_exe(TESTFN) - self.addCleanup(safe_rmpath, TESTFN) - cmdline = [TESTFN] + (["0123456789"] * 20) - sproc = get_test_subprocess(cmdline) - p = psutil.Process(sproc.pid) + testfn = self.get_testfn() + create_exe(testfn) + cmdline = [testfn] + (["0123456789"] * 20) + p = self.spawn_psproc(cmdline) self.assertEqual(p.cmdline(), cmdline) def test_name(self): - sproc = get_test_subprocess(PYTHON_EXE) - name = psutil.Process(sproc.pid).name().lower() + p = self.spawn_psproc(PYTHON_EXE) + name = p.name().lower() pyexe = os.path.basename(os.path.realpath(sys.executable)).lower() assert pyexe.startswith(name), (pyexe, name) @unittest.skipIf(PYPY, "unreliable on PYPY") def test_long_name(self): - long_name = TESTFN + ("0123456789" * 2) - create_exe(long_name) - self.addCleanup(safe_rmpath, long_name) - sproc = get_test_subprocess(long_name) - p = psutil.Process(sproc.pid) - self.assertEqual(p.name(), os.path.basename(long_name)) + testfn = self.get_testfn(suffix="0123456789" * 2) + create_exe(testfn) + p = self.spawn_psproc(testfn) + self.assertEqual(p.name(), os.path.basename(testfn)) # XXX @unittest.skipIf(SUNOS, "broken on SUNOS") @@ -759,27 +755,12 @@ class TestProcess(unittest.TestCase): # Test that name(), exe() and cmdline() correctly handle programs # with funky chars such as spaces and ")", see: # https://github.com/giampaolo/psutil/issues/628 - - def rm(): - # Try to limit occasional failures on Appveyor: - # https://ci.appveyor.com/project/giampaolo/psutil/build/1350/ - # job/lbo3bkju55le850n - try: - safe_rmpath(funky_path) - except OSError: - pass - - funky_path = TESTFN + 'foo bar )' + funky_path = self.get_testfn(suffix='foo bar )') create_exe(funky_path) - self.addCleanup(rm) cmdline = [funky_path, "-c", "import time; [time.sleep(0.01) for x in range(3000)];" "arg1", "arg2", "", "arg3", ""] - sproc = get_test_subprocess(cmdline) - p = psutil.Process(sproc.pid) - # ...in order to try to prevent occasional failures on travis - if TRAVIS: - wait_for_pid(p.pid) + p = self.spawn_psproc(cmdline) self.assertEqual(p.cmdline(), cmdline) self.assertEqual(p.name(), os.path.basename(funky_path)) self.assertEqual(os.path.normcase(p.exe()), @@ -816,51 +797,53 @@ class TestProcess(unittest.TestCase): def test_nice(self): p = psutil.Process() self.assertRaises(TypeError, p.nice, "str") - if WINDOWS: - try: - init = p.nice() - if sys.version_info > (3, 4): - self.assertIsInstance(init, enum.IntEnum) - else: - self.assertIsInstance(init, int) - self.assertEqual(init, psutil.NORMAL_PRIORITY_CLASS) - p.nice(psutil.HIGH_PRIORITY_CLASS) - self.assertEqual(p.nice(), psutil.HIGH_PRIORITY_CLASS) - p.nice(psutil.NORMAL_PRIORITY_CLASS) - self.assertEqual(p.nice(), psutil.NORMAL_PRIORITY_CLASS) - finally: - p.nice(psutil.NORMAL_PRIORITY_CLASS) - else: - first_nice = p.nice() - try: - if hasattr(os, "getpriority"): - self.assertEqual( - os.getpriority(os.PRIO_PROCESS, os.getpid()), p.nice()) - p.nice(1) - self.assertEqual(p.nice(), 1) - if hasattr(os, "getpriority"): - self.assertEqual( - os.getpriority(os.PRIO_PROCESS, os.getpid()), p.nice()) - # XXX - going back to previous nice value raises - # AccessDenied on MACOS - if not MACOS: - p.nice(0) - self.assertEqual(p.nice(), 0) - except psutil.AccessDenied: - pass - finally: + init = p.nice() + try: + if WINDOWS: + for prio in [psutil.NORMAL_PRIORITY_CLASS, + psutil.IDLE_PRIORITY_CLASS, + psutil.BELOW_NORMAL_PRIORITY_CLASS, + psutil.REALTIME_PRIORITY_CLASS, + psutil.HIGH_PRIORITY_CLASS, + psutil.ABOVE_NORMAL_PRIORITY_CLASS]: + with self.subTest(prio=prio): + try: + p.nice(prio) + except psutil.AccessDenied: + pass + else: + self.assertEqual(p.nice(), prio) + else: try: - p.nice(first_nice) + if hasattr(os, "getpriority"): + self.assertEqual( + os.getpriority(os.PRIO_PROCESS, os.getpid()), + p.nice()) + p.nice(1) + self.assertEqual(p.nice(), 1) + if hasattr(os, "getpriority"): + self.assertEqual( + os.getpriority(os.PRIO_PROCESS, os.getpid()), + p.nice()) + # XXX - going back to previous nice value raises + # AccessDenied on MACOS + if not MACOS: + p.nice(0) + self.assertEqual(p.nice(), 0) except psutil.AccessDenied: pass + finally: + try: + p.nice(init) + except psutil.AccessDenied: + pass def test_status(self): p = psutil.Process() self.assertEqual(p.status(), psutil.STATUS_RUNNING) def test_username(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() username = p.username() if WINDOWS: domain, username = username.split('\\') @@ -871,15 +854,13 @@ class TestProcess(unittest.TestCase): self.assertEqual(username, getpass.getuser()) def test_cwd(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() self.assertEqual(p.cwd(), os.getcwd()) def test_cwd_2(self): cmd = [PYTHON_EXE, "-c", "import os, time; os.chdir('..'); time.sleep(60)"] - sproc = get_test_subprocess(cmd) - p = psutil.Process(sproc.pid) + p = self.spawn_psproc(cmd) call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') @@ -894,9 +875,7 @@ class TestProcess(unittest.TestCase): self.assertEqual(len(initial), len(set(initial))) all_cpus = list(range(len(psutil.cpu_percent(percpu=True)))) - # Work around travis failure: - # https://travis-ci.org/giampaolo/psutil/builds/284173194 - for n in all_cpus if not TRAVIS else initial: + for n in all_cpus: p.cpu_affinity([n]) self.assertEqual(p.cpu_affinity(), [n]) if hasattr(os, "sched_getaffinity"): @@ -921,14 +900,12 @@ class TestProcess(unittest.TestCase): self.assertRaises(TypeError, p.cpu_affinity, 1) p.cpu_affinity(initial) # it should work with all iterables, not only lists - if not TRAVIS: - p.cpu_affinity(set(all_cpus)) - p.cpu_affinity(tuple(all_cpus)) + p.cpu_affinity(set(all_cpus)) + p.cpu_affinity(tuple(all_cpus)) @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') def test_cpu_affinity_errs(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() invalid_cpu = [len(psutil.cpu_times(percpu=True)) + 10] self.assertRaises(ValueError, p.cpu_affinity, invalid_cpu) self.assertRaises(ValueError, p.cpu_affinity, range(10000, 11000)) @@ -946,50 +923,49 @@ class TestProcess(unittest.TestCase): if len(initial) > 12: initial = initial[:12] # ...otherwise it will take forever combos = [] - for l in range(0, len(initial) + 1): - for subset in itertools.combinations(initial, l): + for i in range(0, len(initial) + 1): + for subset in itertools.combinations(initial, i): if subset: combos.append(list(subset)) for combo in combos: p.cpu_affinity(combo) - self.assertEqual(p.cpu_affinity(), combo) + self.assertEqual(sorted(p.cpu_affinity()), sorted(combo)) # TODO: #595 @unittest.skipIf(BSD, "broken on BSD") # can't find any process file on Appveyor @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") def test_open_files(self): - # current process p = psutil.Process() + testfn = self.get_testfn() files = p.open_files() - self.assertFalse(TESTFN in files) - with open(TESTFN, 'wb') as f: + self.assertNotIn(testfn, files) + with open(testfn, 'wb') as f: f.write(b'x' * 1024) f.flush() # give the kernel some time to see the new file files = call_until(p.open_files, "len(ret) != %i" % len(files)) filenames = [os.path.normcase(x.path) for x in files] - self.assertIn(os.path.normcase(TESTFN), filenames) + self.assertIn(os.path.normcase(testfn), filenames) if LINUX: for file in files: - if file.path == TESTFN: + if file.path == testfn: self.assertEqual(file.position, 1024) for file in files: assert os.path.isfile(file.path), file # another process - cmdline = "import time; f = open(r'%s', 'r'); time.sleep(60);" % TESTFN - sproc = get_test_subprocess([PYTHON_EXE, "-c", cmdline]) - p = psutil.Process(sproc.pid) + cmdline = "import time; f = open(r'%s', 'r'); time.sleep(60);" % testfn + p = self.spawn_psproc([PYTHON_EXE, "-c", cmdline]) for x in range(100): filenames = [os.path.normcase(x.path) for x in p.open_files()] - if TESTFN in filenames: + if testfn in filenames: break time.sleep(.01) else: - self.assertIn(os.path.normcase(TESTFN), filenames) + self.assertIn(os.path.normcase(testfn), filenames) for file in filenames: assert os.path.isfile(file), file @@ -999,9 +975,10 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") def test_open_files_2(self): # test fd and path fields + p = psutil.Process() normcase = os.path.normcase - with open(TESTFN, 'w') as fileobj: - p = psutil.Process() + testfn = self.get_testfn() + with open(testfn, 'w') as fileobj: for file in p.open_files(): if normcase(file.path) == normcase(fileobj.name) or \ file.fd == fileobj.fileno(): @@ -1023,8 +1000,9 @@ class TestProcess(unittest.TestCase): @unittest.skipIf(not POSIX, 'POSIX only') def test_num_fds(self): p = psutil.Process() + testfn = self.get_testfn() start = p.num_fds() - file = open(TESTFN, 'w') + file = open(testfn, 'w') self.addCleanup(file.close) self.assertEqual(p.num_fds(), start + 1) sock = socket.socket() @@ -1046,83 +1024,73 @@ class TestProcess(unittest.TestCase): self.fail("num ctx switches still the same after 50.000 iterations") def test_ppid(self): + p = psutil.Process() if hasattr(os, 'getppid'): - self.assertEqual(psutil.Process().ppid(), os.getppid()) - this_parent = os.getpid() - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - self.assertEqual(p.ppid(), this_parent) - # no other process is supposed to have us as parent - reap_children(recursive=True) + self.assertEqual(p.ppid(), os.getppid()) + p = self.spawn_psproc() + self.assertEqual(p.ppid(), os.getpid()) if APPVEYOR: # Occasional failures, see: # https://ci.appveyor.com/project/giampaolo/psutil/build/ # job/0hs623nenj7w4m33 return - for p in psutil.process_iter(): - if p.pid == sproc.pid: - continue - # XXX: sometimes this fails on Windows; not sure why. - self.assertNotEqual(p.ppid(), this_parent, msg=p) def test_parent(self): - this_parent = os.getpid() - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - self.assertEqual(p.parent().pid, this_parent) + p = self.spawn_psproc() + self.assertEqual(p.parent().pid, os.getpid()) lowest_pid = psutil.pids()[0] self.assertIsNone(psutil.Process(lowest_pid).parent()) def test_parent_multi(self): - p1, p2 = create_proc_children_pair() - self.assertEqual(p2.parent(), p1) - self.assertEqual(p1.parent(), psutil.Process()) + parent = psutil.Process() + child, grandchild = self.spawn_children_pair() + self.assertEqual(grandchild.parent(), child) + self.assertEqual(child.parent(), parent) def test_parent_disappeared(self): # Emulate a case where the parent process disappeared. - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() with mock.patch("psutil.Process", side_effect=psutil.NoSuchProcess(0, 'foo')): self.assertIsNone(p.parent()) @retry_on_failure() def test_parents(self): - assert psutil.Process().parents() - p1, p2 = create_proc_children_pair() - self.assertEqual(p1.parents()[0], psutil.Process()) - self.assertEqual(p2.parents()[0], p1) - self.assertEqual(p2.parents()[1], psutil.Process()) + parent = psutil.Process() + assert parent.parents() + child, grandchild = self.spawn_children_pair() + self.assertEqual(child.parents()[0], parent) + self.assertEqual(grandchild.parents()[0], child) + self.assertEqual(grandchild.parents()[1], parent) def test_children(self): - reap_children(recursive=True) - p = psutil.Process() - self.assertEqual(p.children(), []) - self.assertEqual(p.children(recursive=True), []) + parent = psutil.Process() + self.assertEqual(parent.children(), []) + self.assertEqual(parent.children(recursive=True), []) # On Windows we set the flag to 0 in order to cancel out the # CREATE_NO_WINDOW flag (enabled by default) which creates # an extra "conhost.exe" child. - sproc = get_test_subprocess(creationflags=0) - children1 = p.children() - children2 = p.children(recursive=True) + child = self.spawn_psproc(creationflags=0) + children1 = parent.children() + children2 = parent.children(recursive=True) for children in (children1, children2): self.assertEqual(len(children), 1) - self.assertEqual(children[0].pid, sproc.pid) - self.assertEqual(children[0].ppid(), os.getpid()) + self.assertEqual(children[0].pid, child.pid) + self.assertEqual(children[0].ppid(), parent.pid) def test_children_recursive(self): # Test children() against two sub processes, p1 and p2, where # p1 (our child) spawned p2 (our grandchild). - p1, p2 = create_proc_children_pair() - p = psutil.Process() - self.assertEqual(p.children(), [p1]) - self.assertEqual(p.children(recursive=True), [p1, p2]) + parent = psutil.Process() + child, grandchild = self.spawn_children_pair() + self.assertEqual(parent.children(), [child]) + self.assertEqual(parent.children(recursive=True), [child, grandchild]) # If the intermediate process is gone there's no way for # children() to recursively find it. - p1.terminate() - p1.wait() - self.assertEqual(p.children(recursive=True), []) + child.terminate() + child.wait() + self.assertEqual(parent.children(recursive=True), []) def test_children_duplicates(self): # find the process which has the highest number of children @@ -1134,6 +1102,8 @@ class TestProcess(unittest.TestCase): pass # this is the one, now let's make sure there are no duplicates pid = sorted(table.items(), key=lambda x: x[1])[-1][0] + if LINUX and pid == 0: + raise self.skipTest("PID 0") p = psutil.Process(pid) try: c = p.children(recursive=True) @@ -1143,21 +1113,20 @@ class TestProcess(unittest.TestCase): self.assertEqual(len(c), len(set(c))) def test_parents_and_children(self): - p1, p2 = create_proc_children_pair() - me = psutil.Process() + parent = psutil.Process() + child, grandchild = self.spawn_children_pair() # forward - children = me.children(recursive=True) + children = parent.children(recursive=True) self.assertEqual(len(children), 2) - self.assertEqual(children[0], p1) - self.assertEqual(children[1], p2) + self.assertEqual(children[0], child) + self.assertEqual(children[1], grandchild) # backward - parents = p2.parents() - self.assertEqual(parents[0], p1) - self.assertEqual(parents[1], me) + parents = grandchild.parents() + self.assertEqual(parents[0], child) + self.assertEqual(parents[1], parent) def test_suspend_resume(self): - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) + p = self.spawn_psproc() p.suspend() for x in range(100): if p.status() == psutil.STATUS_STOPPED: @@ -1217,8 +1186,8 @@ class TestProcess(unittest.TestCase): p.as_dict(['foo', 'bar']) def test_oneshot(self): + p = psutil.Process() with mock.patch("psutil._psplatform.Process.cpu_times") as m: - p = psutil.Process() with p.oneshot(): p.cpu_times() p.cpu_times() @@ -1232,9 +1201,9 @@ class TestProcess(unittest.TestCase): def test_oneshot_twice(self): # Test the case where the ctx manager is __enter__ed twice. # The second __enter__ is supposed to resut in a NOOP. + p = psutil.Process() with mock.patch("psutil._psplatform.Process.cpu_times") as m1: with mock.patch("psutil._psplatform.Process.oneshot_enter") as m2: - p = psutil.Process() with p.oneshot(): p.cpu_times() p.cpu_times() @@ -1253,7 +1222,7 @@ class TestProcess(unittest.TestCase): # Make sure oneshot() cache is nonglobal. Instead it's # supposed to be bound to the Process instance, see: # https://github.com/giampaolo/psutil/issues/1373 - p1, p2 = create_proc_children_pair() + p1, p2 = self.spawn_children_pair() p1_ppid = p1.ppid() p2_ppid = p2.ppid() self.assertNotEqual(p1_ppid, p2_ppid) @@ -1272,131 +1241,58 @@ class TestProcess(unittest.TestCase): # >>> time.sleep(2) # time-consuming task, process dies in meantime # >>> proc.name() # Refers to Issue #15 - sproc = get_test_subprocess() - p = psutil.Process(sproc.pid) - p.terminate() - p.wait() - if WINDOWS: - call_until(psutil.pids, "%s not in ret" % p.pid) - assert not p.is_running() - - if WINDOWS: - with self.assertRaises(psutil.NoSuchProcess): - p.send_signal(signal.CTRL_C_EVENT) - with self.assertRaises(psutil.NoSuchProcess): - p.send_signal(signal.CTRL_BREAK_EVENT) - - excluded_names = ['pid', 'is_running', 'wait', 'create_time', - 'oneshot', 'memory_info_ex'] - if LINUX and not HAS_RLIMIT: - excluded_names.append('rlimit') - for name in dir(p): - if (name.startswith('_') or - name in excluded_names): - continue + def assert_raises_nsp(fun, fun_name): try: - meth = getattr(p, name) - # get/set methods - if name == 'nice': - if POSIX: - ret = meth(1) - else: - ret = meth(psutil.NORMAL_PRIORITY_CLASS) - elif name == 'ionice': - ret = meth() - ret = meth(2) - elif name == 'rlimit': - ret = meth(psutil.RLIMIT_NOFILE) - ret = meth(psutil.RLIMIT_NOFILE, (5, 5)) - elif name == 'cpu_affinity': - ret = meth() - ret = meth([0]) - elif name == 'send_signal': - ret = meth(signal.SIGTERM) - else: - ret = meth() - except psutil.ZombieProcess: - self.fail("ZombieProcess for %r was not supposed to happen" % - name) + ret = fun() + except psutil.ZombieProcess: # differentiate from NSP + raise except psutil.NoSuchProcess: pass except psutil.AccessDenied: - if OPENBSD and name in ('threads', 'num_threads'): - pass - else: - raise - except NotImplementedError: - pass + if OPENBSD and fun_name in ('threads', 'num_threads'): + return + raise else: - # NtQuerySystemInformation succeeds if process is gone. - if WINDOWS and name in ('exe', 'name'): - normcase = os.path.normcase - if name == 'exe': - self.assertEqual(normcase(ret), normcase(PYTHON_EXE)) - else: - self.assertEqual( - normcase(ret), - normcase(os.path.basename(PYTHON_EXE))) - continue - self.fail( - "NoSuchProcess exception not raised for %r, retval=%s" % ( - name, ret)) + # NtQuerySystemInformation succeeds even if process is gone. + if WINDOWS and fun_name in ('exe', 'name'): + return + raise self.fail("%r didn't raise NSP and returned %r " + "instead" % (fun, ret)) + + p = self.spawn_psproc() + p.terminate() + p.wait() + if WINDOWS: # XXX + call_until(psutil.pids, "%s not in ret" % p.pid) + self.assertProcessGone(p) + + ns = process_namespace(p) + for fun, name in ns.iter(ns.all): + assert_raises_nsp(fun, name) + + # NtQuerySystemInformation succeeds even if process is gone. + if WINDOWS and not GITHUB_ACTIONS: + normcase = os.path.normcase + self.assertEqual(normcase(p.exe()), normcase(PYTHON_EXE)) @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process(self): - def succeed_or_zombie_p_exc(fun, *args, **kwargs): + def succeed_or_zombie_p_exc(fun): try: - return fun(*args, **kwargs) + return fun() except (psutil.ZombieProcess, psutil.AccessDenied): pass - zpid = create_zombie_proc() - self.addCleanup(reap_children, recursive=True) + parent, zombie = self.spawn_zombie() # A zombie process should always be instantiable - zproc = psutil.Process(zpid) + zproc = psutil.Process(zombie.pid) # ...and at least its status always be querable self.assertEqual(zproc.status(), psutil.STATUS_ZOMBIE) # ...and it should be considered 'running' - self.assertTrue(zproc.is_running()) + assert zproc.is_running() # ...and as_dict() shouldn't crash zproc.as_dict() - # if cmdline succeeds it should be an empty list - ret = succeed_or_zombie_p_exc(zproc.suspend) - if ret is not None: - self.assertEqual(ret, []) - - if hasattr(zproc, "rlimit"): - succeed_or_zombie_p_exc(zproc.rlimit, psutil.RLIMIT_NOFILE) - succeed_or_zombie_p_exc(zproc.rlimit, psutil.RLIMIT_NOFILE, - (5, 5)) - # set methods - succeed_or_zombie_p_exc(zproc.parent) - if hasattr(zproc, 'cpu_affinity'): - try: - succeed_or_zombie_p_exc(zproc.cpu_affinity, [0]) - except ValueError as err: - if TRAVIS and LINUX and "not eligible" in str(err): - # https://travis-ci.org/giampaolo/psutil/jobs/279890461 - pass - else: - raise - - succeed_or_zombie_p_exc(zproc.nice, 0) - if hasattr(zproc, 'ionice'): - if LINUX: - succeed_or_zombie_p_exc(zproc.ionice, 2, 0) - else: - succeed_or_zombie_p_exc(zproc.ionice, 0) # Windows - if hasattr(zproc, 'rlimit'): - succeed_or_zombie_p_exc(zproc.rlimit, - psutil.RLIMIT_NOFILE, (5, 5)) - succeed_or_zombie_p_exc(zproc.suspend) - succeed_or_zombie_p_exc(zproc.resume) - succeed_or_zombie_p_exc(zproc.terminate) - succeed_or_zombie_p_exc(zproc.kill) - - # ...its parent should 'see' it - # edit: not true on BSD and MACOS + # ...its parent should 'see' it (edit: not true on BSD and MACOS # descendants = [x.pid for x in psutil.Process().children( # recursive=True)] # self.assertIn(zpid, descendants) @@ -1405,15 +1301,16 @@ class TestProcess(unittest.TestCase): # rid of a zombie is to kill its parent. # self.assertEqual(zpid.ppid(), os.getpid()) # ...and all other APIs should be able to deal with it - self.assertTrue(psutil.pid_exists(zpid)) - if not TRAVIS and MACOS: - # For some reason this started failing all of the sudden. - # Maybe they upgraded MACOS version? - # https://travis-ci.org/giampaolo/psutil/jobs/310896404 - self.assertIn(zpid, psutil.pids()) - self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) - psutil._pmap = {} - self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) + + ns = process_namespace(zproc) + for fun, name in ns.iter(ns.all): + succeed_or_zombie_p_exc(fun) + + assert psutil.pid_exists(zproc.pid) + self.assertIn(zproc.pid, psutil.pids()) + self.assertIn(zproc.pid, [x.pid for x in psutil.process_iter()]) + psutil._pmap = {} + self.assertIn(zproc.pid, [x.pid for x in psutil.process_iter()]) @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process_is_running_w_exc(self): @@ -1435,50 +1332,63 @@ class TestProcess(unittest.TestCase): self.assertEqual(p.status(), psutil.STATUS_ZOMBIE) assert m.called + def test_reused_pid(self): + # Emulate a case where PID has been reused by another process. + subp = self.spawn_testproc() + p = psutil.Process(subp.pid) + p._ident = (p.pid, p.create_time() + 100) + assert not p.is_running() + assert p != psutil.Process(subp.pid) + msg = "process no longer exists and its PID has been reused" + self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.suspend) + self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.resume) + self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.terminate) + self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.kill) + self.assertRaisesRegex(psutil.NoSuchProcess, msg, p.children) + def test_pid_0(self): # Process(0) is supposed to work on all platforms except Linux if 0 not in psutil.pids(): self.assertRaises(psutil.NoSuchProcess, psutil.Process, 0) + # These 2 are a contradiction, but "ps" says PID 1's parent + # is PID 0. + assert not psutil.pid_exists(0) + self.assertEqual(psutil.Process(1).ppid(), 0) return - # test all methods p = psutil.Process(0) - for name in psutil._as_dict_attrnames: - if name == 'pid': - continue - meth = getattr(p, name) + exc = psutil.AccessDenied if WINDOWS else ValueError + self.assertRaises(exc, p.wait) + self.assertRaises(exc, p.terminate) + self.assertRaises(exc, p.suspend) + self.assertRaises(exc, p.resume) + self.assertRaises(exc, p.kill) + self.assertRaises(exc, p.send_signal, signal.SIGTERM) + + # test all methods + ns = process_namespace(p) + for fun, name in ns.iter(ns.getters + ns.setters): try: - ret = meth() + ret = fun() except psutil.AccessDenied: pass else: if name in ("uids", "gids"): self.assertEqual(ret.real, 0) elif name == "username": - if POSIX: - self.assertEqual(p.username(), 'root') - elif WINDOWS: - self.assertEqual(p.username(), 'NT AUTHORITY\\SYSTEM') + user = 'NT AUTHORITY\\SYSTEM' if WINDOWS else 'root' + self.assertEqual(p.username(), user) elif name == "name": assert name, name - if hasattr(p, 'rlimit'): - try: - p.rlimit(psutil.RLIMIT_FSIZE) - except psutil.AccessDenied: - pass - - p.as_dict() - if not OPENBSD: self.assertIn(0, psutil.pids()) - self.assertTrue(psutil.pid_exists(0)) + assert psutil.pid_exists(0) @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): def clean_dict(d): # Most of these are problematic on Travis. - d.pop("PSUTIL_TESTING", None) d.pop("PLAT", None) d.pop("HOME", None) if MACOS: @@ -1494,7 +1404,8 @@ class TestProcess(unittest.TestCase): p = psutil.Process() d1 = clean_dict(p.environ()) d2 = clean_dict(os.environ.copy()) - self.assertEqual(d1, d2) + if not OSX and GITHUB_ACTIONS: + self.assertEqual(d1, d2) @unittest.skipIf(not HAS_ENVIRON, "not supported") @unittest.skipIf(not POSIX, "POSIX only") @@ -1513,18 +1424,25 @@ class TestProcess(unittest.TestCase): return execve("/bin/cat", argv, envp); } """) - path = TESTFN + path = self.get_testfn() create_exe(path, c_code=code) - self.addCleanup(safe_rmpath, path) - sproc = get_test_subprocess([path], - stdin=subprocess.PIPE, - stderr=subprocess.PIPE) + sproc = self.spawn_testproc( + [path], stdin=subprocess.PIPE, stderr=subprocess.PIPE) p = psutil.Process(sproc.pid) wait_for_pid(p.pid) - self.assertTrue(p.is_running()) + assert p.is_running() # Wait for process to exec or exit. self.assertEqual(sproc.stderr.read(), b"") - self.assertEqual(p.environ(), {"A": "1", "C": "3"}) + if MACOS and CI_TESTING: + try: + env = p.environ() + except psutil.AccessDenied: + # XXX: fails sometimes with: + # PermissionError from 'sysctl(KERN_PROCARGS2) -> EIO' + return + else: + env = p.environ() + self.assertEqual(env, {"A": "1", "C": "3"}) sproc.communicate() self.assertEqual(sproc.returncode, 0) @@ -1535,6 +1453,7 @@ class TestProcess(unittest.TestCase): if POSIX and os.getuid() == 0: + class LimitedUserTestCase(TestProcess): """Repeat the previous tests by using a limited user. Executed only on UNIX and only if the user who run the test script @@ -1546,7 +1465,7 @@ if POSIX and os.getuid() == 0: PROCESS_GID = os.getgid() def __init__(self, *args, **kwargs): - TestProcess.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) # re-define all existent test methods in order to # ignore AccessDenied exceptions for attr in [x for x in dir(self) if x.startswith('test')]: @@ -1560,15 +1479,14 @@ if POSIX and os.getuid() == 0: setattr(self, attr, types.MethodType(test_, self)) def setUp(self): - safe_rmpath(TESTFN) - TestProcess.setUp(self) + super().setUp() os.setegid(1000) os.seteuid(1000) def tearDown(self): os.setegid(self.PROCESS_UID) os.seteuid(self.PROCESS_GID) - TestProcess.tearDown(self) + super().tearDown() def test_nice(self): try: @@ -1578,8 +1496,8 @@ if POSIX and os.getuid() == 0: else: self.fail("exception not raised") + @unittest.skipIf(1, "causes problem as root") def test_zombie_process(self): - # causes problems if test test suite is run as root pass @@ -1588,10 +1506,11 @@ if POSIX and os.getuid() == 0: # =================================================================== -class TestPopen(unittest.TestCase): +class TestPopen(PsutilTestCase): """Tests for psutil.Popen class.""" - def tearDown(self): + @classmethod + def tearDownClass(cls): reap_children() def test_misc(self): @@ -1607,6 +1526,10 @@ class TestPopen(unittest.TestCase): self.assertTrue(dir(proc)) self.assertRaises(AttributeError, getattr, proc, 'foo') proc.terminate() + if POSIX: + self.assertEqual(proc.wait(5), -signal.SIGTERM) + else: + self.assertEqual(proc.wait(5), signal.SIGTERM) def test_ctx_manager(self): with psutil.Popen([PYTHON_EXE, "-V"], @@ -1640,5 +1563,5 @@ class TestPopen(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_sunos.py b/psutil/tests/test_sunos.py index 94405d41..ad94f774 100755 --- a/psutil/tests/test_sunos.py +++ b/psutil/tests/test_sunos.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -10,12 +10,13 @@ import os import psutil from psutil import SUNOS +from psutil.tests import PsutilTestCase from psutil.tests import sh from psutil.tests import unittest @unittest.skipIf(not SUNOS, "SUNOS only") -class SunOSSpecificTestCase(unittest.TestCase): +class SunOSSpecificTestCase(PsutilTestCase): def test_swap_memory(self): out = sh('env PATH=/usr/sbin:/sbin:%s swap -l' % os.environ['PATH']) @@ -41,5 +42,5 @@ class SunOSSpecificTestCase(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index e1ce042a..b6516bef 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -15,7 +15,6 @@ import shutil import signal import socket import sys -import tempfile import time import psutil @@ -36,7 +35,7 @@ from psutil.tests import check_net_address from psutil.tests import CI_TESTING from psutil.tests import DEVNULL from psutil.tests import enum -from psutil.tests import get_test_subprocess +from psutil.tests import GLOBAL_TIMEOUT from psutil.tests import HAS_BATTERY from psutil.tests import HAS_CPU_FREQ from psutil.tests import HAS_DISK_SWAPS @@ -45,14 +44,13 @@ from psutil.tests import HAS_NET_IO_COUNTERS from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import IS_64BIT from psutil.tests import mock +from psutil.tests import PsutilTestCase from psutil.tests import PYPY -from psutil.tests import reap_children from psutil.tests import retry_on_failure -from psutil.tests import safe_rmpath -from psutil.tests import TESTFN -from psutil.tests import TESTFN_UNICODE -from psutil.tests import TRAVIS +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import UNICODE_SUFFIX from psutil.tests import unittest @@ -61,17 +59,11 @@ from psutil.tests import unittest # =================================================================== -class TestProcessAPIs(unittest.TestCase): - - def setUp(self): - safe_rmpath(TESTFN) - - def tearDown(self): - reap_children() +class TestProcessAPIs(PsutilTestCase): def test_process_iter(self): self.assertIn(os.getpid(), [x.pid for x in psutil.process_iter()]) - sproc = get_test_subprocess() + sproc = self.spawn_testproc() self.assertIn(sproc.pid, [x.pid for x in psutil.process_iter()]) p = psutil.Process(sproc.pid) p.kill() @@ -106,32 +98,16 @@ class TestProcessAPIs(unittest.TestCase): self.assertGreaterEqual(p.info['pid'], 0) assert m.called - def test_process_iter_new_only(self): - ls1 = list(psutil.process_iter(attrs=['pid'])) - ls2 = list(psutil.process_iter(attrs=['pid'], new_only=True)) - self.assertGreater(len(ls1), len(ls2)) - # assume no more than 3 new processes were created in the meantime - self.assertIn(len(ls2), [0, 1, 2, 3, 4, 5]) - - sproc = get_test_subprocess() - ls = list(psutil.process_iter(attrs=['pid'], new_only=True)) - self.assertIn(len(ls2), [0, 1, 2, 3, 4, 5]) - for p in ls: - if p.pid == sproc.pid: - break - else: - self.fail("subprocess not found") - @unittest.skipIf(PYPY and WINDOWS, - "get_test_subprocess() unreliable on PYPY + WINDOWS") + "spawn_testproc() unreliable on PYPY + WINDOWS") def test_wait_procs(self): def callback(p): pids.append(p.pid) pids = [] - sproc1 = get_test_subprocess() - sproc2 = get_test_subprocess() - sproc3 = get_test_subprocess() + sproc1 = self.spawn_testproc() + sproc2 = self.spawn_testproc() + sproc3 = self.spawn_testproc() procs = [psutil.Process(x.pid) for x in (sproc1, sproc2, sproc3)] self.assertRaises(ValueError, psutil.wait_procs, procs, timeout=-1) self.assertRaises(TypeError, psutil.wait_procs, procs, callback=1) @@ -180,18 +156,18 @@ class TestProcessAPIs(unittest.TestCase): self.assertTrue(hasattr(p, 'returncode')) @unittest.skipIf(PYPY and WINDOWS, - "get_test_subprocess() unreliable on PYPY + WINDOWS") + "spawn_testproc() unreliable on PYPY + WINDOWS") def test_wait_procs_no_timeout(self): - sproc1 = get_test_subprocess() - sproc2 = get_test_subprocess() - sproc3 = get_test_subprocess() + sproc1 = self.spawn_testproc() + sproc2 = self.spawn_testproc() + sproc3 = self.spawn_testproc() procs = [psutil.Process(x.pid) for x in (sproc1, sproc2, sproc3)] for p in procs: p.terminate() gone, alive = psutil.wait_procs(procs) def test_pid_exists(self): - sproc = get_test_subprocess() + sproc = self.spawn_testproc() self.assertTrue(psutil.pid_exists(sproc.pid)) p = psutil.Process(sproc.pid) p.kill() @@ -201,7 +177,6 @@ class TestProcessAPIs(unittest.TestCase): self.assertEqual(psutil.pid_exists(0), 0 in psutil.pids()) def test_pid_exists_2(self): - reap_children() pids = psutil.pids() for pid in pids: try: @@ -216,15 +191,8 @@ class TestProcessAPIs(unittest.TestCase): for pid in pids: self.assertFalse(psutil.pid_exists(pid), msg=pid) - def test_pids(self): - pidslist = psutil.pids() - procslist = [x.pid for x in psutil.process_iter()] - # make sure every pid is unique - self.assertEqual(sorted(set(pidslist)), pidslist) - self.assertEqual(pidslist, procslist) - -class TestMiscAPIs(unittest.TestCase): +class TestMiscAPIs(PsutilTestCase): def test_boot_time(self): bt = psutil.boot_time() @@ -251,14 +219,6 @@ class TestMiscAPIs(unittest.TestCase): else: psutil.Process(user.pid) - @unittest.skipIf(not POSIX, 'POSIX only') - def test_PAGESIZE(self): - # pagesize is used internally to perform different calculations - # and it's determined by using SC_PAGE_SIZE; make sure - # getpagesize() returns the same value. - import resource - self.assertEqual(os.sysconf("SC_PAGE_SIZE"), resource.getpagesize()) - def test_test(self): # test for psutil.test() function stdout = sys.stdout @@ -306,7 +266,7 @@ class TestMiscAPIs(unittest.TestCase): self.assertIs(getattr(psutil, name), False, msg=name) -class TestMemoryAPIs(unittest.TestCase): +class TestMemoryAPIs(PsutilTestCase): def test_virtual_memory(self): mem = psutil.virtual_memory() @@ -343,7 +303,7 @@ class TestMemoryAPIs(unittest.TestCase): assert mem.sout >= 0, mem -class TestCpuAPIs(unittest.TestCase): +class TestCpuAPIs(PsutilTestCase): def test_cpu_count_logical(self): logical = psutil.cpu_count() @@ -357,16 +317,16 @@ class TestCpuAPIs(unittest.TestCase): if "physical id" not in cpuinfo_data: raise unittest.SkipTest("cpuinfo doesn't include physical id") - def test_cpu_count_physical(self): + def test_cpu_count_cores(self): logical = psutil.cpu_count() - physical = psutil.cpu_count(logical=False) - if physical is None: - raise self.skipTest("physical cpu_count() is None") + cores = psutil.cpu_count(logical=False) + if cores is None: + raise self.skipTest("cpu_count_cores() is None") if WINDOWS and sys.getwindowsversion()[:2] <= (6, 1): # <= Vista - self.assertIsNone(physical) + self.assertIsNone(cores) else: - self.assertGreaterEqual(physical, 1) - self.assertGreaterEqual(logical, physical) + self.assertGreaterEqual(cores, 1) + self.assertGreaterEqual(logical, cores) def test_cpu_count_none(self): # https://github.com/giampaolo/psutil/issues/1085 @@ -375,7 +335,7 @@ class TestCpuAPIs(unittest.TestCase): return_value=val) as m: self.assertIsNone(psutil.cpu_count()) assert m.called - with mock.patch('psutil._psplatform.cpu_count_physical', + with mock.patch('psutil._psplatform.cpu_count_cores', return_value=val) as m: self.assertIsNone(psutil.cpu_count(logical=False)) assert m.called @@ -412,7 +372,7 @@ class TestCpuAPIs(unittest.TestCase): def test_cpu_times_time_increases(self): # Make sure time increases between calls. t1 = sum(psutil.cpu_times()) - stop_at = time.time() + 1 + stop_at = time.time() + GLOBAL_TIMEOUT while time.time() < stop_at: t2 = sum(psutil.cpu_times()) if t2 > t1: @@ -456,7 +416,7 @@ class TestCpuAPIs(unittest.TestCase): # Simulate some work load then make sure time have increased # between calls. tot1 = psutil.cpu_times(percpu=True) - giveup_at = time.time() + 1 + giveup_at = time.time() + GLOBAL_TIMEOUT while True: if time.time() >= giveup_at: return self.fail("timeout") @@ -515,6 +475,8 @@ class TestCpuAPIs(unittest.TestCase): self._test_cpu_percent(percent, last, new) self._test_cpu_percent(sum(new), last, new) last = new + with self.assertRaises(ValueError): + psutil.cpu_times_percent(interval=-1) def test_per_cpu_times_percent(self): last = psutil.cpu_times_percent(interval=0.001, percpu=True) @@ -563,8 +525,6 @@ class TestCpuAPIs(unittest.TestCase): self.assertGreaterEqual(value, 0) ls = psutil.cpu_freq(percpu=True) - if TRAVIS and not ls: - raise self.skipTest("skipped on Travis") if FREEBSD and not ls: raise self.skipTest("returns empty list on FreeBSD") @@ -577,15 +537,15 @@ class TestCpuAPIs(unittest.TestCase): @unittest.skipIf(not HAS_GETLOADAVG, "not supported") def test_getloadavg(self): loadavg = psutil.getloadavg() - assert len(loadavg) == 3 - + self.assertEqual(len(loadavg), 3) for load in loadavg: self.assertIsInstance(load, float) self.assertGreaterEqual(load, 0.0) -class TestDiskAPIs(unittest.TestCase): +class TestDiskAPIs(PsutilTestCase): + @unittest.skipIf(PYPY and not IS_64BIT, "unreliable on PYPY32 + 32BIT") def test_disk_usage(self): usage = psutil.disk_usage(os.getcwd()) self.assertEqual(usage._fields, ('total', 'used', 'free', 'percent')) @@ -608,31 +568,37 @@ class TestDiskAPIs(unittest.TestCase): # if path does not exist OSError ENOENT is expected across # all platforms - fname = tempfile.mktemp() + fname = self.get_testfn() with self.assertRaises(FileNotFoundError): psutil.disk_usage(fname) + @unittest.skipIf(not ASCII_FS, "not an ASCII fs") def test_disk_usage_unicode(self): # See: https://github.com/giampaolo/psutil/issues/416 - if ASCII_FS: - with self.assertRaises(UnicodeEncodeError): - psutil.disk_usage(TESTFN_UNICODE) + with self.assertRaises(UnicodeEncodeError): + psutil.disk_usage(UNICODE_SUFFIX) def test_disk_usage_bytes(self): psutil.disk_usage(b'.') def test_disk_partitions(self): + def check_ntuple(nt): + self.assertIsInstance(nt.device, str) + self.assertIsInstance(nt.mountpoint, str) + self.assertIsInstance(nt.fstype, str) + self.assertIsInstance(nt.opts, str) + self.assertIsInstance(nt.maxfile, (int, type(None))) + self.assertIsInstance(nt.maxpath, (int, type(None))) + if nt.maxfile is not None and not GITHUB_ACTIONS: + self.assertGreater(nt.maxfile, 0) + if nt.maxpath is not None: + self.assertGreater(nt.maxpath, 0) + # all = False ls = psutil.disk_partitions(all=False) - # on travis we get: - # self.assertEqual(p.cpu_affinity(), [n]) - # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, 7,... != [0] self.assertTrue(ls, msg=ls) for disk in ls: - self.assertIsInstance(disk.device, str) - self.assertIsInstance(disk.mountpoint, str) - self.assertIsInstance(disk.fstype, str) - self.assertIsInstance(disk.opts, str) + check_ntuple(disk) if WINDOWS and 'cdrom' in disk.opts: continue if not POSIX: @@ -649,11 +615,12 @@ class TestDiskAPIs(unittest.TestCase): ls = psutil.disk_partitions(all=True) self.assertTrue(ls, msg=ls) for disk in psutil.disk_partitions(all=True): + check_ntuple(disk) if not WINDOWS and disk.mountpoint: try: os.stat(disk.mountpoint) except OSError as err: - if TRAVIS and MACOS and err.errno == errno.EIO: + if GITHUB_ACTIONS and MACOS and err.errno == errno.EIO: continue # http://mail.python.org/pipermail/python-dev/ # 2012-June/120787.html @@ -661,8 +628,8 @@ class TestDiskAPIs(unittest.TestCase): raise else: assert os.path.exists(disk.mountpoint), disk - self.assertIsInstance(disk.fstype, str) - self.assertIsInstance(disk.opts, str) + + # --- def find_mount_point(path): path = os.path.abspath(path) @@ -674,7 +641,6 @@ class TestDiskAPIs(unittest.TestCase): mounts = [x.mountpoint.lower() for x in psutil.disk_partitions(all=True) if x.mountpoint] self.assertIn(mount, mounts) - psutil.disk_usage(mount) @unittest.skipIf(LINUX and not os.path.exists('/proc/diskstats'), '/proc/diskstats not available on this linux version') @@ -747,7 +713,7 @@ class TestDiskAPIs(unittest.TestCase): sum([x.used for x in psutil.disk_swaps()])) -class TestNetAPIs(unittest.TestCase): +class TestNetAPIs(PsutilTestCase): @unittest.skipIf(not HAS_NET_IO_COUNTERS, 'not supported') def test_net_io_counters(self): @@ -809,7 +775,7 @@ class TestNetAPIs(unittest.TestCase): self.assertIsInstance(addr.netmask, (str, type(None))) self.assertIsInstance(addr.broadcast, (str, type(None))) self.assertIn(addr.family, families) - if sys.version_info >= (3, 4): + if sys.version_info >= (3, 4) and not PYPY: self.assertIsInstance(addr.family, enum.IntEnum) if nic_stats[nic].isup: # Do not test binding to addresses of interfaces @@ -865,7 +831,6 @@ class TestNetAPIs(unittest.TestCase): else: self.assertEqual(addr.address, '06-3d-29-00-00-00') - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") # raises EPERM def test_net_if_stats(self): nics = psutil.net_if_stats() assert nics, nics @@ -892,7 +857,7 @@ class TestNetAPIs(unittest.TestCase): assert m.called -class TestSensorsAPIs(unittest.TestCase): +class TestSensorsAPIs(PsutilTestCase): @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_sensors_temperatures(self): @@ -946,5 +911,5 @@ class TestSensorsAPIs(unittest.TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_testutils.py b/psutil/tests/test_testutils.py new file mode 100755 index 00000000..09adbdb1 --- /dev/null +++ b/psutil/tests/test_testutils.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Tests for testing utils (psutil.tests namespace). +""" + +import collections +import contextlib +import errno +import os +import socket +import stat +import subprocess + +from psutil import FREEBSD +from psutil import NETBSD +from psutil import POSIX +from psutil._common import open_binary +from psutil._common import open_text +from psutil._common import supports_ipv6 +from psutil.tests import bind_socket +from psutil.tests import bind_unix_socket +from psutil.tests import call_until +from psutil.tests import chdir +from psutil.tests import CI_TESTING +from psutil.tests import create_sockets +from psutil.tests import get_free_port +from psutil.tests import HAS_CONNECTIONS_UNIX +from psutil.tests import is_namedtuple +from psutil.tests import mock +from psutil.tests import process_namespace +from psutil.tests import PsutilTestCase +from psutil.tests import PYTHON_EXE +from psutil.tests import reap_children +from psutil.tests import retry +from psutil.tests import retry_on_failure +from psutil.tests import safe_mkdir +from psutil.tests import safe_rmpath +from psutil.tests import serialrun +from psutil.tests import system_namespace +from psutil.tests import tcp_socketpair +from psutil.tests import terminate +from psutil.tests import TestMemoryLeak +from psutil.tests import unittest +from psutil.tests import unix_socketpair +from psutil.tests import wait_for_file +from psutil.tests import wait_for_pid +import psutil +import psutil.tests + +# =================================================================== +# --- Unit tests for test utilities. +# =================================================================== + + +class TestRetryDecorator(PsutilTestCase): + + @mock.patch('time.sleep') + def test_retry_success(self, sleep): + # Fail 3 times out of 5; make sure the decorated fun returns. + + @retry(retries=5, interval=1, logfun=None) + def foo(): + while queue: + queue.pop() + 1 / 0 + return 1 + + queue = list(range(3)) + self.assertEqual(foo(), 1) + self.assertEqual(sleep.call_count, 3) + + @mock.patch('time.sleep') + def test_retry_failure(self, sleep): + # Fail 6 times out of 5; th function is supposed to raise exc. + @retry(retries=5, interval=1, logfun=None) + def foo(): + while queue: + queue.pop() + 1 / 0 + return 1 + + queue = list(range(6)) + self.assertRaises(ZeroDivisionError, foo) + self.assertEqual(sleep.call_count, 5) + + @mock.patch('time.sleep') + def test_exception_arg(self, sleep): + @retry(exception=ValueError, interval=1) + def foo(): + raise TypeError + + self.assertRaises(TypeError, foo) + self.assertEqual(sleep.call_count, 0) + + @mock.patch('time.sleep') + def test_no_interval_arg(self, sleep): + # if interval is not specified sleep is not supposed to be called + + @retry(retries=5, interval=None, logfun=None) + def foo(): + 1 / 0 + + self.assertRaises(ZeroDivisionError, foo) + self.assertEqual(sleep.call_count, 0) + + @mock.patch('time.sleep') + def test_retries_arg(self, sleep): + + @retry(retries=5, interval=1, logfun=None) + def foo(): + 1 / 0 + + self.assertRaises(ZeroDivisionError, foo) + self.assertEqual(sleep.call_count, 5) + + @mock.patch('time.sleep') + def test_retries_and_timeout_args(self, sleep): + self.assertRaises(ValueError, retry, retries=5, timeout=1) + + +class TestSyncTestUtils(PsutilTestCase): + + def test_wait_for_pid(self): + wait_for_pid(os.getpid()) + nopid = max(psutil.pids()) + 99999 + with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): + self.assertRaises(psutil.NoSuchProcess, wait_for_pid, nopid) + + def test_wait_for_file(self): + testfn = self.get_testfn() + with open(testfn, 'w') as f: + f.write('foo') + wait_for_file(testfn) + assert not os.path.exists(testfn) + + def test_wait_for_file_empty(self): + testfn = self.get_testfn() + with open(testfn, 'w'): + pass + wait_for_file(testfn, empty=True) + assert not os.path.exists(testfn) + + def test_wait_for_file_no_file(self): + testfn = self.get_testfn() + with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): + self.assertRaises(IOError, wait_for_file, testfn) + + def test_wait_for_file_no_delete(self): + testfn = self.get_testfn() + with open(testfn, 'w') as f: + f.write('foo') + wait_for_file(testfn, delete=False) + assert os.path.exists(testfn) + + def test_call_until(self): + ret = call_until(lambda: 1, "ret == 1") + self.assertEqual(ret, 1) + + +class TestFSTestUtils(PsutilTestCase): + + def test_open_text(self): + with open_text(__file__) as f: + self.assertEqual(f.mode, 'rt') + + def test_open_binary(self): + with open_binary(__file__) as f: + self.assertEqual(f.mode, 'rb') + + def test_safe_mkdir(self): + testfn = self.get_testfn() + safe_mkdir(testfn) + assert os.path.isdir(testfn) + safe_mkdir(testfn) + assert os.path.isdir(testfn) + + def test_safe_rmpath(self): + # test file is removed + testfn = self.get_testfn() + open(testfn, 'w').close() + safe_rmpath(testfn) + assert not os.path.exists(testfn) + # test no exception if path does not exist + safe_rmpath(testfn) + # test dir is removed + os.mkdir(testfn) + safe_rmpath(testfn) + assert not os.path.exists(testfn) + # test other exceptions are raised + with mock.patch('psutil.tests.os.stat', + side_effect=OSError(errno.EINVAL, "")) as m: + with self.assertRaises(OSError): + safe_rmpath(testfn) + assert m.called + + def test_chdir(self): + testfn = self.get_testfn() + base = os.getcwd() + os.mkdir(testfn) + with chdir(testfn): + self.assertEqual(os.getcwd(), os.path.join(base, testfn)) + self.assertEqual(os.getcwd(), base) + + +class TestProcessUtils(PsutilTestCase): + + def test_reap_children(self): + subp = self.spawn_testproc() + p = psutil.Process(subp.pid) + assert p.is_running() + reap_children() + assert not p.is_running() + assert not psutil.tests._pids_started + assert not psutil.tests._subprocesses_started + + def test_spawn_children_pair(self): + child, grandchild = self.spawn_children_pair() + self.assertNotEqual(child.pid, grandchild.pid) + assert child.is_running() + assert grandchild.is_running() + children = psutil.Process().children() + self.assertEqual(children, [child]) + children = psutil.Process().children(recursive=True) + self.assertEqual(len(children), 2) + self.assertIn(child, children) + self.assertIn(grandchild, children) + self.assertEqual(child.ppid(), os.getpid()) + self.assertEqual(grandchild.ppid(), child.pid) + + terminate(child) + assert not child.is_running() + assert grandchild.is_running() + + terminate(grandchild) + assert not grandchild.is_running() + + @unittest.skipIf(not POSIX, "POSIX only") + def test_spawn_zombie(self): + parent, zombie = self.spawn_zombie() + self.assertEqual(zombie.status(), psutil.STATUS_ZOMBIE) + + def test_terminate(self): + # by subprocess.Popen + p = self.spawn_testproc() + terminate(p) + self.assertProcessGone(p) + terminate(p) + # by psutil.Process + p = psutil.Process(self.spawn_testproc().pid) + terminate(p) + self.assertProcessGone(p) + terminate(p) + # by psutil.Popen + cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] + p = psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + terminate(p) + self.assertProcessGone(p) + terminate(p) + # by PID + pid = self.spawn_testproc().pid + terminate(pid) + self.assertProcessGone(p) + terminate(pid) + # zombie + if POSIX: + parent, zombie = self.spawn_zombie() + terminate(parent) + terminate(zombie) + self.assertProcessGone(parent) + self.assertProcessGone(zombie) + + +class TestNetUtils(PsutilTestCase): + + def bind_socket(self): + port = get_free_port() + with contextlib.closing(bind_socket(addr=('', port))) as s: + self.assertEqual(s.getsockname()[1], port) + + @unittest.skipIf(not POSIX, "POSIX only") + def test_bind_unix_socket(self): + name = self.get_testfn() + sock = bind_unix_socket(name) + with contextlib.closing(sock): + self.assertEqual(sock.family, socket.AF_UNIX) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.getsockname(), name) + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + # UDP + name = self.get_testfn() + sock = bind_unix_socket(name, type=socket.SOCK_DGRAM) + with contextlib.closing(sock): + self.assertEqual(sock.type, socket.SOCK_DGRAM) + + def tcp_tcp_socketpair(self): + addr = ("127.0.0.1", get_free_port()) + server, client = tcp_socketpair(socket.AF_INET, addr=addr) + with contextlib.closing(server): + with contextlib.closing(client): + # Ensure they are connected and the positions are + # correct. + self.assertEqual(server.getsockname(), addr) + self.assertEqual(client.getpeername(), addr) + self.assertNotEqual(client.getsockname(), addr) + + @unittest.skipIf(not POSIX, "POSIX only") + @unittest.skipIf(NETBSD or FREEBSD, + "/var/run/log UNIX socket opened by default") + def test_unix_socketpair(self): + p = psutil.Process() + num_fds = p.num_fds() + assert not p.connections(kind='unix') + name = self.get_testfn() + server, client = unix_socketpair(name) + try: + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + self.assertEqual(p.num_fds() - num_fds, 2) + self.assertEqual(len(p.connections(kind='unix')), 2) + self.assertEqual(server.getsockname(), name) + self.assertEqual(client.getpeername(), name) + finally: + client.close() + server.close() + + def test_create_sockets(self): + with create_sockets() as socks: + fams = collections.defaultdict(int) + types = collections.defaultdict(int) + for s in socks: + fams[s.family] += 1 + # work around http://bugs.python.org/issue30204 + types[s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)] += 1 + self.assertGreaterEqual(fams[socket.AF_INET], 2) + if supports_ipv6(): + self.assertGreaterEqual(fams[socket.AF_INET6], 2) + if POSIX and HAS_CONNECTIONS_UNIX: + self.assertGreaterEqual(fams[socket.AF_UNIX], 2) + self.assertGreaterEqual(types[socket.SOCK_STREAM], 2) + self.assertGreaterEqual(types[socket.SOCK_DGRAM], 2) + + +@serialrun +class TestMemLeakClass(TestMemoryLeak): + + @retry_on_failure() + def test_times(self): + def fun(): + cnt['cnt'] += 1 + cnt = {'cnt': 0} + self.execute(fun, times=10, warmup_times=15) + self.assertEqual(cnt['cnt'], 26) + + def test_param_err(self): + self.assertRaises(ValueError, self.execute, lambda: 0, times=0) + self.assertRaises(ValueError, self.execute, lambda: 0, times=-1) + self.assertRaises(ValueError, self.execute, lambda: 0, warmup_times=-1) + self.assertRaises(ValueError, self.execute, lambda: 0, tolerance=-1) + self.assertRaises(ValueError, self.execute, lambda: 0, retries=-1) + + @retry_on_failure() + @unittest.skipIf(CI_TESTING, "skipped on CI") + def test_leak_mem(self): + ls = [] + + def fun(ls=ls): + ls.append("x" * 24 * 1024) + + try: + # will consume around 3M in total + self.assertRaisesRegex(AssertionError, "extra-mem", + self.execute, fun, times=50) + finally: + del ls + + def test_unclosed_files(self): + def fun(): + f = open(__file__) + self.addCleanup(f.close) + box.append(f) + + box = [] + kind = "fd" if POSIX else "handle" + self.assertRaisesRegex(AssertionError, "unclosed " + kind, + self.execute, fun) + + def test_tolerance(self): + def fun(): + ls.append("x" * 24 * 1024) + ls = [] + times = 100 + self.execute(fun, times=times, warmup_times=0, + tolerance=200 * 1024 * 1024) + self.assertEqual(len(ls), times + 1) + + def test_execute_w_exc(self): + def fun(): + 1 / 0 + self.execute_w_exc(ZeroDivisionError, fun) + with self.assertRaises(ZeroDivisionError): + self.execute_w_exc(OSError, fun) + + def fun(): + pass + with self.assertRaises(AssertionError): + self.execute_w_exc(ZeroDivisionError, fun) + + +class TestTestingUtils(PsutilTestCase): + + def test_process_namespace(self): + p = psutil.Process() + ns = process_namespace(p) + ns.test() + fun = [x for x in ns.iter(ns.getters) if x[1] == 'ppid'][0][0] + self.assertEqual(fun(), p.ppid()) + + def test_system_namespace(self): + ns = system_namespace() + fun = [x for x in ns.iter(ns.getters) if x[1] == 'net_if_addrs'][0][0] + self.assertEqual(fun(), psutil.net_if_addrs()) + + +class TestOtherUtils(PsutilTestCase): + + def test_is_namedtuple(self): + assert is_namedtuple(collections.namedtuple('foo', 'a b c')(1, 2, 3)) + assert not is_namedtuple(tuple()) + + +if __name__ == '__main__': + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 81a28807..9edb8c89 100644..100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. @@ -74,44 +74,45 @@ etc.) and make sure that: """ import os +import shutil import traceback import warnings from contextlib import closing from psutil import BSD -from psutil import MACOS from psutil import OPENBSD from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 from psutil._compat import u from psutil.tests import APPVEYOR -from psutil.tests import CIRRUS from psutil.tests import ASCII_FS from psutil.tests import bind_unix_socket from psutil.tests import chdir +from psutil.tests import CI_TESTING from psutil.tests import copyload_shared_lib from psutil.tests import create_exe -from psutil.tests import get_test_subprocess +from psutil.tests import get_testfn from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import INVALID_UNICODE_SUFFIX +from psutil.tests import PsutilTestCase from psutil.tests import PYPY -from psutil.tests import reap_children from psutil.tests import safe_mkdir -from psutil.tests import safe_rmpath as _safe_rmpath +from psutil.tests import safe_rmpath +from psutil.tests import serialrun from psutil.tests import skip_on_access_denied -from psutil.tests import TESTFILE_PREFIX -from psutil.tests import TESTFN -from psutil.tests import TESTFN_UNICODE -from psutil.tests import TRAVIS +from psutil.tests import spawn_testproc +from psutil.tests import terminate +from psutil.tests import TESTFN_PREFIX +from psutil.tests import UNICODE_SUFFIX from psutil.tests import unittest -from psutil.tests import unix_socket_path import psutil -def safe_rmpath(path): - if APPVEYOR: +if APPVEYOR: + def safe_rmpath(path): # NOQA # TODO - this is quite random and I'm not sure why it happens, # nor I can reproduce it locally: # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ @@ -122,38 +123,33 @@ def safe_rmpath(path): # https://github.com/giampaolo/psutil/blob/ # 68c7a70728a31d8b8b58f4be6c4c0baa2f449eda/psutil/arch/ # windows/process_info.c#L146 + from psutil.tests import safe_rmpath as rm try: - return _safe_rmpath(path) + return rm(path) except WindowsError: traceback.print_exc() - else: - return _safe_rmpath(path) -def subprocess_supports_unicode(name): +def try_unicode(suffix): """Return True if both the fs and the subprocess module can deal with a unicode file name. """ - if PY3: - return True + sproc = None + testfn = get_testfn(suffix=suffix) try: - safe_rmpath(name) - create_exe(name) - get_test_subprocess(cmd=[name]) - except UnicodeEncodeError: + safe_rmpath(testfn) + create_exe(testfn) + sproc = spawn_testproc(cmd=[testfn]) + shutil.copyfile(testfn, testfn + '-2') + safe_rmpath(testfn + '-2') + except (UnicodeEncodeError, IOError): return False else: return True finally: - reap_children() - - -# An invalid unicode string. -if PY3: - INVALID_NAME = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( - 'utf8', 'surrogateescape') -else: - INVALID_NAME = TESTFN + "f\xc0\x80" + if sproc is not None: + terminate(sproc) + safe_rmpath(testfn) # =================================================================== @@ -161,27 +157,44 @@ else: # =================================================================== -class _BaseFSAPIsTests(object): - funky_name = None +class BaseUnicodeTest(PsutilTestCase): + funky_suffix = None + + def setUp(self): + if self.funky_suffix is not None: + if not try_unicode(self.funky_suffix): + raise self.skipTest("can't handle unicode str") + + +@serialrun +@unittest.skipIf(ASCII_FS, "ASCII fs") +@unittest.skipIf(PYPY and not PY3, "too much trouble on PYPY2") +class TestFSAPIs(BaseUnicodeTest): + """Test FS APIs with a funky, valid, UTF8 path name.""" + + funky_suffix = UNICODE_SUFFIX @classmethod def setUpClass(cls): - safe_rmpath(cls.funky_name) + cls.funky_name = get_testfn(suffix=cls.funky_suffix) create_exe(cls.funky_name) @classmethod def tearDownClass(cls): - reap_children() safe_rmpath(cls.funky_name) - def tearDown(self): - reap_children() - def expect_exact_path_match(self): - raise NotImplementedError("must be implemented in subclass") + # Do not expect psutil to correctly handle unicode paths on + # Python 2 if os.listdir() is not able either. + here = '.' if isinstance(self.funky_name, str) else u('.') + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return self.funky_name in os.listdir(here) + + # --- def test_proc_exe(self): - subp = get_test_subprocess(cmd=[self.funky_name]) + subp = self.spawn_testproc(cmd=[self.funky_name]) p = psutil.Process(subp.pid) exe = p.exe() self.assertIsInstance(exe, str) @@ -190,14 +203,14 @@ class _BaseFSAPIsTests(object): os.path.normcase(self.funky_name)) def test_proc_name(self): - subp = get_test_subprocess(cmd=[self.funky_name]) + subp = self.spawn_testproc(cmd=[self.funky_name]) name = psutil.Process(subp.pid).name() self.assertIsInstance(name, str) if self.expect_exact_path_match(): self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): - subp = get_test_subprocess(cmd=[self.funky_name]) + subp = self.spawn_testproc(cmd=[self.funky_name]) p = psutil.Process(subp.pid) cmdline = p.cmdline() for part in cmdline: @@ -233,21 +246,20 @@ class _BaseFSAPIsTests(object): @unittest.skipIf(not POSIX, "POSIX only") def test_proc_connections(self): - suffix = os.path.basename(self.funky_name) - with unix_socket_path(suffix=suffix) as name: - try: - sock = bind_unix_socket(name) - except UnicodeEncodeError: - if PY3: - raise - else: - raise unittest.SkipTest("not supported") - with closing(sock): - conn = psutil.Process().connections('unix')[0] - self.assertIsInstance(conn.laddr, str) - # AF_UNIX addr not set on OpenBSD - if not OPENBSD and not CIRRUS: # XXX - self.assertEqual(conn.laddr, name) + name = self.get_testfn(suffix=self.funky_suffix) + try: + sock = bind_unix_socket(name) + except UnicodeEncodeError: + if PY3: + raise + else: + raise unittest.SkipTest("not supported") + with closing(sock): + conn = psutil.Process().connections('unix')[0] + self.assertIsInstance(conn.laddr, str) + # AF_UNIX addr not set on OpenBSD + if not OPENBSD: # XXX + self.assertEqual(conn.laddr, name) @unittest.skipIf(not POSIX, "POSIX only") @unittest.skipIf(not HAS_CONNECTIONS_UNIX, "can't list UNIX sockets") @@ -255,26 +267,25 @@ class _BaseFSAPIsTests(object): def test_net_connections(self): def find_sock(cons): for conn in cons: - if os.path.basename(conn.laddr).startswith(TESTFILE_PREFIX): + if os.path.basename(conn.laddr).startswith(TESTFN_PREFIX): return conn raise ValueError("connection not found") - suffix = os.path.basename(self.funky_name) - with unix_socket_path(suffix=suffix) as name: - try: - sock = bind_unix_socket(name) - except UnicodeEncodeError: - if PY3: - raise - else: - raise unittest.SkipTest("not supported") - with closing(sock): - cons = psutil.net_connections(kind='unix') - # AF_UNIX addr not set on OpenBSD - if not OPENBSD: - conn = find_sock(cons) - self.assertIsInstance(conn.laddr, str) - self.assertEqual(conn.laddr, name) + name = self.get_testfn(suffix=self.funky_suffix) + try: + sock = bind_unix_socket(name) + except UnicodeEncodeError: + if PY3: + raise + else: + raise unittest.SkipTest("not supported") + with closing(sock): + cons = psutil.net_connections(kind='unix') + # AF_UNIX addr not set on OpenBSD + if not OPENBSD: + conn = find_sock(cons) + self.assertIsInstance(conn.laddr, str) + self.assertEqual(conn.laddr, name) def test_disk_usage(self): dname = self.funky_name + "2" @@ -284,51 +295,26 @@ class _BaseFSAPIsTests(object): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") @unittest.skipIf(not PY3, "ctypes does not support unicode on PY2") - @unittest.skipIf(PYPY and WINDOWS, - "copyload_shared_lib() unsupported on PYPY + WINDOWS") + @unittest.skipIf(PYPY, "unstable on PYPY") def test_memory_maps(self): # XXX: on Python 2, using ctypes.CDLL with a unicode path # opens a message box which blocks the test run. - with copyload_shared_lib(dst_prefix=self.funky_name) as funky_path: + with copyload_shared_lib(suffix=self.funky_suffix) as funky_path: def normpath(p): return os.path.realpath(os.path.normcase(p)) libpaths = [normpath(x.path) for x in psutil.Process().memory_maps()] # ...just to have a clearer msg in case of failure - libpaths = [x for x in libpaths if TESTFILE_PREFIX in x] + libpaths = [x for x in libpaths if TESTFN_PREFIX in x] self.assertIn(normpath(funky_path), libpaths) for path in libpaths: self.assertIsInstance(path, str) -# https://travis-ci.org/giampaolo/psutil/jobs/440073249 -@unittest.skipIf(PYPY and TRAVIS, "unreliable on PYPY + TRAVIS") -@unittest.skipIf(MACOS and TRAVIS, "unreliable on TRAVIS") # TODO -@unittest.skipIf(ASCII_FS, "ASCII fs") -@unittest.skipIf(not subprocess_supports_unicode(TESTFN_UNICODE), - "subprocess can't deal with unicode") -class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): - """Test FS APIs with a funky, valid, UTF8 path name.""" - funky_name = TESTFN_UNICODE - - @classmethod - def expect_exact_path_match(cls): - # Do not expect psutil to correctly handle unicode paths on - # Python 2 if os.listdir() is not able either. - here = '.' if isinstance(cls.funky_name, str) else u('.') - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return cls.funky_name in os.listdir(here) - - -@unittest.skipIf(PYPY and TRAVIS, "unreliable on PYPY + TRAVIS") -@unittest.skipIf(MACOS and TRAVIS, "unreliable on TRAVIS") # TODO -@unittest.skipIf(PYPY, "unreliable on PYPY") -@unittest.skipIf(not subprocess_supports_unicode(INVALID_NAME), - "subprocess can't deal with invalid unicode") -class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): +@unittest.skipIf(CI_TESTING, "unreliable on CI") +class TestFSAPIsWithInvalidPath(TestFSAPIs): """Test FS APIs with a funky, invalid path name.""" - funky_name = INVALID_NAME + funky_suffix = INVALID_UNICODE_SUFFIX @classmethod def expect_exact_path_match(cls): @@ -341,11 +327,9 @@ class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): # =================================================================== -class TestNonFSAPIS(unittest.TestCase): +class TestNonFSAPIS(BaseUnicodeTest): """Unicode tests for non fs-related APIs.""" - - def tearDown(self): - reap_children() + funky_suffix = UNICODE_SUFFIX if PY3 else 'è' @unittest.skipIf(not HAS_ENVIRON, "not supported") @unittest.skipIf(PYPY and WINDOWS, "segfaults on PYPY + WINDOWS") @@ -356,17 +340,16 @@ class TestNonFSAPIS(unittest.TestCase): # we use "è", which is part of the extended ASCII table # (unicode point <= 255). env = os.environ.copy() - funky_str = TESTFN_UNICODE if PY3 else 'è' - env['FUNNY_ARG'] = funky_str - sproc = get_test_subprocess(env=env) + env['FUNNY_ARG'] = self.funky_suffix + sproc = self.spawn_testproc(env=env) p = psutil.Process(sproc.pid) env = p.environ() for k, v in env.items(): self.assertIsInstance(k, str) self.assertIsInstance(v, str) - self.assertEqual(env['FUNNY_ARG'], funky_str) + self.assertEqual(env['FUNNY_ARG'], self.funky_suffix) if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 934ea847..d9b65adb 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: UTF-8 -* # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. @@ -22,49 +22,43 @@ import warnings import psutil from psutil import WINDOWS from psutil._compat import FileNotFoundError +from psutil._compat import super from psutil.tests import APPVEYOR -from psutil.tests import get_test_subprocess +from psutil.tests import GITHUB_ACTIONS from psutil.tests import HAS_BATTERY +from psutil.tests import IS_64BIT from psutil.tests import mock +from psutil.tests import PsutilTestCase from psutil.tests import PY3 from psutil.tests import PYPY -from psutil.tests import reap_children from psutil.tests import retry_on_failure from psutil.tests import sh +from psutil.tests import spawn_testproc +from psutil.tests import terminate +from psutil.tests import TOLERANCE_DISK_USAGE from psutil.tests import unittest if WINDOWS and not PYPY: with warnings.catch_warnings(): warnings.simplefilter("ignore") - import win32api # requires "pip install pypiwin32" + import win32api # requires "pip install pywin32" import win32con import win32process import wmi # requires "pip install wmi" / "make setup-dev-env" +if WINDOWS: + from psutil._pswindows import convert_oserror -cext = psutil._psplatform.cext -# are we a 64 bit process -IS_64_BIT = sys.maxsize > 2**32 +cext = psutil._psplatform.cext -def wrap_exceptions(fun): - def wrapper(self, *args, **kwargs): - try: - return fun(self, *args, **kwargs) - except OSError as err: - from psutil._pswindows import ACCESS_DENIED_SET - if err.errno in ACCESS_DENIED_SET: - raise psutil.AccessDenied(None, None) - if err.errno == errno.ESRCH: - raise psutil.NoSuchProcess(None, None) - raise - return wrapper - - -@unittest.skipIf(PYPY, "pywin32 not available on PYPY") # skip whole module -class TestCase(unittest.TestCase): +@unittest.skipIf(not WINDOWS, "WINDOWS only") +@unittest.skipIf(PYPY, "pywin32 not available on PYPY") +# https://github.com/giampaolo/psutil/pull/1762#issuecomment-632892692 +@unittest.skipIf(GITHUB_ACTIONS and not PY3, "pywin32 broken on GITHUB + PY2") +class WindowsTestCase(PsutilTestCase): pass @@ -73,8 +67,7 @@ class TestCase(unittest.TestCase): # =================================================================== -@unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestCpuAPIs(TestCase): +class TestCpuAPIs(WindowsTestCase): @unittest.skipIf('NUMBER_OF_PROCESSORS' not in os.environ, 'NUMBER_OF_PROCESSORS env var is not available') @@ -96,7 +89,7 @@ class TestCpuAPIs(TestCase): proc = w.Win32_Processor()[0] self.assertEqual(psutil.cpu_count(), proc.NumberOfLogicalProcessors) - def test_cpu_count_phys_vs_wmi(self): + def test_cpu_count_cores_vs_wmi(self): w = wmi.WMI() proc = w.Win32_Processor()[0] self.assertEqual(psutil.cpu_count(logical=False), proc.NumberOfCores) @@ -112,8 +105,7 @@ class TestCpuAPIs(TestCase): self.assertEqual(proc.MaxClockSpeed, psutil.cpu_freq().max) -@unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestSystemAPIs(TestCase): +class TestSystemAPIs(WindowsTestCase): def test_nic_names(self): out = sh('ipconfig /all') @@ -164,6 +156,8 @@ class TestSystemAPIs(TestCase): break if 'cdrom' in ps_part.opts: break + if ps_part.mountpoint.startswith('A:'): + break # floppy try: usage = psutil.disk_usage(ps_part.mountpoint) except FileNotFoundError: @@ -180,6 +174,7 @@ class TestSystemAPIs(TestCase): else: self.fail("can't find partition %s" % repr(ps_part)) + @retry_on_failure() def test_disk_usage(self): for disk in psutil.disk_partitions(): if 'cdrom' in disk.opts: @@ -187,9 +182,9 @@ class TestSystemAPIs(TestCase): sys_value = win32api.GetDiskFreeSpaceEx(disk.mountpoint) psutil_value = psutil.disk_usage(disk.mountpoint) self.assertAlmostEqual(sys_value[0], psutil_value.free, - delta=1024 * 1024) + delta=TOLERANCE_DISK_USAGE) self.assertAlmostEqual(sys_value[1], psutil_value.total, - delta=1024 * 1024) + delta=TOLERANCE_DISK_USAGE) self.assertEqual(psutil_value.used, psutil_value.total - psutil_value.free) @@ -197,7 +192,8 @@ class TestSystemAPIs(TestCase): sys_value = [ x + '\\' for x in win32api.GetLogicalDriveStrings().split("\\\x00") if x and not x.startswith('A:')] - psutil_value = [x.mountpoint for x in psutil.disk_partitions(all=True)] + psutil_value = [x.mountpoint for x in psutil.disk_partitions(all=True) + if not x.mountpoint.startswith('A:')] self.assertEqual(sys_value, psutil_value) def test_net_if_stats(self): @@ -236,8 +232,7 @@ class TestSystemAPIs(TestCase): # =================================================================== -@unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestSensorsBattery(TestCase): +class TestSensorsBattery(WindowsTestCase): def test_has_battery(self): if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: @@ -297,16 +292,15 @@ class TestSensorsBattery(TestCase): # =================================================================== -@unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestProcess(TestCase): +class TestProcess(WindowsTestCase): @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess().pid + cls.pid = spawn_testproc().pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) def test_issue_24(self): p = psutil.Process(0) @@ -343,46 +337,10 @@ class TestProcess(TestCase): win32api.CloseHandle(handle) self.assertEqual(p.num_handles(), before) - def test_handles_leak(self): - # Call all Process methods and make sure no handles are left - # open. This is here mainly to make sure functions using - # OpenProcess() always call CloseHandle(). - def call(p, attr): - attr = getattr(p, name, None) - if attr is not None and callable(attr): - attr() - else: - attr - - p = psutil.Process(self.pid) - failures = [] - for name in dir(psutil.Process): - if name.startswith('_') \ - or name in ('terminate', 'kill', 'suspend', 'resume', - 'nice', 'send_signal', 'wait', 'children', - 'as_dict', 'memory_info_ex'): - continue - else: - try: - call(p, name) - num1 = p.num_handles() - call(p, name) - num2 = p.num_handles() - except (psutil.NoSuchProcess, psutil.AccessDenied): - pass - else: - if num2 > num1: - fail = \ - "failure while processing Process.%s method " \ - "(before=%s, after=%s)" % (name, num1, num2) - failures.append(fail) - if failures: - self.fail('\n' + '\n'.join(failures)) - @unittest.skipIf(not sys.version_info >= (2, 7), "CTRL_* signals not supported") def test_ctrl_signals(self): - p = psutil.Process(get_test_subprocess().pid) + p = psutil.Process(self.spawn_testproc().pid) p.send_signal(signal.CTRL_C_EVENT) p.send_signal(signal.CTRL_BREAK_EVENT) p.kill() @@ -525,23 +483,24 @@ class TestProcess(TestCase): self.assertRaises(psutil.NoSuchProcess, proc.exe) -@unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestProcessWMI(TestCase): +class TestProcessWMI(WindowsTestCase): """Compare Process API results with WMI.""" @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess().pid + cls.pid = spawn_testproc().pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) def test_name(self): w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] p = psutil.Process(self.pid) self.assertEqual(p.name(), w.Caption) + # This fail on github because using virtualenv for test environment + @unittest.skipIf(GITHUB_ACTIONS, "unreliable path on GITHUB_ACTIONS") def test_exe(self): w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] p = psutil.Process(self.pid) @@ -562,15 +521,15 @@ class TestProcessWMI(TestCase): username = "%s\\%s" % (domain, username) self.assertEqual(p.username(), username) + @retry_on_failure() def test_memory_rss(self): - time.sleep(0.1) w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] p = psutil.Process(self.pid) rss = p.memory_info().rss self.assertEqual(rss, int(w.WorkingSetSize)) + @retry_on_failure() def test_memory_vms(self): - time.sleep(0.1) w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] p = psutil.Process(self.pid) vms = p.memory_info().vms @@ -591,8 +550,11 @@ class TestProcessWMI(TestCase): self.assertEqual(wmic_create, psutil_create) +# --- + + @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestDualProcessImplementation(TestCase): +class TestDualProcessImplementation(PsutilTestCase): """ Certain APIs on Windows have 2 internal implementations, one based on documented Windows APIs, another one based @@ -605,11 +567,11 @@ class TestDualProcessImplementation(TestCase): @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess().pid + cls.pid = spawn_testproc().pid @classmethod def tearDownClass(cls): - reap_children() + terminate(cls.pid) def test_memory_info(self): mem_1 = psutil.Process(self.pid).memory_info() @@ -660,7 +622,6 @@ class TestDualProcessImplementation(TestCase): assert fun.called def test_cmdline(self): - from psutil._pswindows import convert_oserror for pid in psutil.pids(): try: a = cext.proc_cmdline(pid, use_peb=True) @@ -675,7 +636,7 @@ class TestDualProcessImplementation(TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class RemoteProcessTestCase(TestCase): +class RemoteProcessTestCase(PsutilTestCase): """Certain functions require calling ReadProcessMemory. This trivially works when called on the current process. Check that this works on other processes, especially when they @@ -694,44 +655,41 @@ class RemoteProcessTestCase(TestCase): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, _ = proc.communicate() - if output == str(not IS_64_BIT): + proc.wait() + if output == str(not IS_64BIT): return filename - @classmethod - def setUpClass(cls): - other_python = cls.find_other_interpreter() + test_args = ["-c", "import sys; sys.stdin.read()"] + + def setUp(self): + super().setUp() + other_python = self.find_other_interpreter() if other_python is None: raise unittest.SkipTest( "could not find interpreter with opposite bitness") - - if IS_64_BIT: - cls.python64 = sys.executable - cls.python32 = other_python + if IS_64BIT: + self.python64 = sys.executable + self.python32 = other_python else: - cls.python64 = other_python - cls.python32 = sys.executable - - test_args = ["-c", "import sys; sys.stdin.read()"] + self.python64 = other_python + self.python32 = sys.executable - def setUp(self): env = os.environ.copy() env["THINK_OF_A_NUMBER"] = str(os.getpid()) - self.proc32 = get_test_subprocess([self.python32] + self.test_args, - env=env, - stdin=subprocess.PIPE) - self.proc64 = get_test_subprocess([self.python64] + self.test_args, - env=env, - stdin=subprocess.PIPE) + self.proc32 = self.spawn_testproc( + [self.python32] + self.test_args, + env=env, + stdin=subprocess.PIPE) + self.proc64 = self.spawn_testproc( + [self.python64] + self.test_args, + env=env, + stdin=subprocess.PIPE) def tearDown(self): + super().tearDown() self.proc32.communicate() self.proc64.communicate() - reap_children() - - @classmethod - def tearDownClass(cls): - reap_children() def test_cmdline_32(self): p = psutil.Process(self.proc32.pid) @@ -755,7 +713,7 @@ class RemoteProcessTestCase(TestCase): p = psutil.Process(self.proc32.pid) e = p.environ() self.assertIn("THINK_OF_A_NUMBER", e) - self.assertEquals(e["THINK_OF_A_NUMBER"], str(os.getpid())) + self.assertEqual(e["THINK_OF_A_NUMBER"], str(os.getpid())) def test_environ_64(self): p = psutil.Process(self.proc64.pid) @@ -771,7 +729,7 @@ class RemoteProcessTestCase(TestCase): @unittest.skipIf(not WINDOWS, "WINDOWS only") -class TestServices(TestCase): +class TestServices(PsutilTestCase): def test_win_service_iter(self): valid_statuses = set([ @@ -866,5 +824,5 @@ class TestServices(TestCase): if __name__ == '__main__': - from psutil.tests.runner import run - run(__file__) + from psutil.tests.runner import run_from_name + run_from_name(__file__) diff --git a/scripts/battery.py b/scripts/battery.py index abbad878..edf4ce8c 100755 --- a/scripts/battery.py +++ b/scripts/battery.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ Show battery information. -$ python scripts/battery.py +$ python3 scripts/battery.py charge: 74% left: 2:11:31 status: discharging diff --git a/scripts/cpu_distribution.py b/scripts/cpu_distribution.py index c509c732..fb39d888 100755 --- a/scripts/cpu_distribution.py +++ b/scripts/cpu_distribution.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ Shows CPU workload split across different CPUs. -$ python scripts/cpu_workload.py +$ python3 scripts/cpu_workload.py CPU 0 CPU 1 CPU 2 CPU 3 CPU 4 CPU 5 CPU 6 CPU 7 19.8 20.6 18.2 15.8 6.9 17.3 5.0 20.4 gvfsd pytho kwork chrom unity kwork kwork kwork @@ -84,7 +84,7 @@ def main(): # processes procs = collections.defaultdict(list) - for p in psutil.process_iter(attrs=['name', 'cpu_num']): + for p in psutil.process_iter(['name', 'cpu_num']): procs[p.info['cpu_num']].append(p.info['name'][:5]) curr_line = 3 diff --git a/scripts/disk_usage.py b/scripts/disk_usage.py index 2edfdbe0..f2c54fc6 100755 --- a/scripts/disk_usage.py +++ b/scripts/disk_usage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ List all mounted disk partitions a-la "df -h" command. -$ python scripts/disk_usage.py +$ python3 scripts/disk_usage.py Device Total Used Free Use % Type Mount /dev/sdb3 18.9G 14.7G 3.3G 77% ext4 / /dev/sda6 345.9G 83.8G 244.5G 24% ext4 /home diff --git a/scripts/fans.py b/scripts/fans.py index 7a0ccf91..179af631 100755 --- a/scripts/fans.py +++ b/scripts/fans.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/scripts/free.py b/scripts/free.py index 82e962ff..8c3359d8 100755 --- a/scripts/free.py +++ b/scripts/free.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ A clone of 'free' cmdline utility. -$ python scripts/free.py +$ python3 scripts/free.py total used free shared buffers cache Mem: 10125520 8625996 1499524 0 349500 3307836 Swap: 0 0 0 diff --git a/scripts/ifconfig.py b/scripts/ifconfig.py index ad62a44f..ae137fb4 100755 --- a/scripts/ifconfig.py +++ b/scripts/ifconfig.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ A clone of 'ifconfig' on UNIX. -$ python scripts/ifconfig.py +$ python3 scripts/ifconfig.py lo: stats : speed=0MB, duplex=?, mtu=65536, up=yes incoming : bytes=1.95M, pkts=22158, errs=0, drops=0 diff --git a/scripts/internal/.git-pre-commit b/scripts/internal/.git-pre-commit deleted file mode 100755 index 621879df..00000000 --- a/scripts/internal/.git-pre-commit +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -""" -This gets executed on 'git commit' and rejects the commit in case the -submitted code does not pass validation. Validation is run only against -the *.py files which were modified in the commit. Checks: - -- assert no space at EOLs -- assert not pdb.set_trace in code -- assert no bare except clause ("except:") in code -- assert "flake8" returns no warnings - -Install this with "make install-git-hooks". -""" - -from __future__ import print_function -import os -import subprocess -import sys - - -def term_supports_colors(): - try: - import curses - assert sys.stderr.isatty() - curses.setupterm() - assert curses.tigetnum("colors") > 0 - except Exception: - return False - else: - return True - - -def hilite(s, ok=True, bold=False): - """Return an highlighted version of 'string'.""" - if not term_supports_colors(): - return s - attr = [] - if ok is None: # no color - pass - elif ok: # green - attr.append('32') - else: # red - attr.append('31') - if bold: - attr.append('1') - return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) - - -def exit(msg): - msg = hilite(msg, ok=False) - print(msg, file=sys.stderr) - sys.exit(1) - - -def sh(cmd): - """run cmd in a subprocess and return its output. - raises RuntimeError on error. - """ - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - stdout, stderr = p.communicate() - if p.returncode != 0: - raise RuntimeError(stderr) - if stderr: - print(stderr, file=sys.stderr) - if stdout.endswith('\n'): - stdout = stdout[:-1] - return stdout - - -def main(): - out = sh("git diff --cached --name-only") - py_files = [x for x in out.split('\n') if x.endswith('.py') and - os.path.exists(x)] - - lineno = 0 - kw = {'encoding': 'utf8'} if sys.version_info[0] == 3 else {} - for path in py_files: - with open(path, 'rt', **kw) as f: - for line in f: - lineno += 1 - # space at end of line - if line.endswith(' '): - print("%s:%s %r" % (path, lineno, line)) - return exit( - "commit aborted: space at end of line") - line = line.rstrip() - # pdb - if "pdb.set_trace" in line: - print("%s:%s %s" % (path, lineno, line)) - return exit( - "commit aborted: you forgot a pdb in your python code") - # bare except clause - if "except:" in line and not line.endswith("# NOQA"): - print("%s:%s %s" % (path, lineno, line)) - return exit("commit aborted: bare except clause") - - # flake8 - if py_files: - try: - import flake8 # NOQA - except ImportError: - return exit("commit aborted: flake8 is not installed; " - "run 'make setup-dev-env'") - - # XXX: we should scape spaces and possibly other amenities here - ret = subprocess.call( - "%s -m flake8 %s" % (sys.executable, " ".join(py_files)), - shell=True) - if ret != 0: - return exit("commit aborted: python code is not flake8 compliant") - - -main() diff --git a/scripts/internal/bench_oneshot.py b/scripts/internal/bench_oneshot.py index 28ad4bac..436bdd6b 100755 --- a/scripts/internal/bench_oneshot.py +++ b/scripts/internal/bench_oneshot.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/scripts/internal/bench_oneshot_2.py b/scripts/internal/bench_oneshot_2.py index becf930c..3867391b 100644..100755 --- a/scripts/internal/bench_oneshot_2.py +++ b/scripts/internal/bench_oneshot_2.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 73134818..e66448fd 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola', Himanshu Shekhar. # All rights reserved. Use of this source code is governed by a @@ -40,7 +40,6 @@ Author: Himanshu Shekhar <https://github.com/himanshub16> (2017) """ from __future__ import print_function - import concurrent.futures import functools import os @@ -175,7 +174,7 @@ def get_urls(fname): return parse_c(fname) else: with open(fname, 'rt', errors='ignore') as f: - if f.readline().strip().startswith('#!/usr/bin/env python'): + if f.readline().strip().startswith('#!/usr/bin/env python3'): return parse_py(fname) return parse_generic(fname) diff --git a/scripts/internal/clinter.py b/scripts/internal/clinter.py new file mode 100755 index 00000000..fde1a3f2 --- /dev/null +++ b/scripts/internal/clinter.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A super simple linter to check C syntax.""" + +from __future__ import print_function +import argparse +import sys + + +warned = False + + +def warn(path, line, lineno, msg): + global warned + warned = True + print("%s:%s: %s" % (path, lineno, msg), file=sys.stderr) + + +def check_line(path, line, idx, lines): + s = line + lineno = idx + 1 + eof = lineno == len(lines) + if s.endswith(' \n'): + warn(path, line, lineno, "extra space at EOL") + elif '\t' in line: + warn(path, line, lineno, "line has a tab") + elif s.endswith('\r\n'): + warn(path, line, lineno, "Windows line ending") + # end of global block, e.g. "}newfunction...": + elif s == "}\n": + if not eof: + nextline = lines[idx + 1] + # "#" is a pre-processor line + if nextline != '\n' and \ + nextline.strip()[0] != '#' and \ + nextline.strip()[:2] != '*/': + warn(path, line, lineno, "expected 1 blank line") + + sls = s.lstrip() + if sls.startswith('//') and sls[2] != ' ' and line.strip() != '//': + warn(path, line, lineno, "no space after // comment") + # e.g. "if(..." after keywords + keywords = ("if", "else", "while", "do", "enum", "for") + for kw in keywords: + if sls.startswith(kw + '('): + warn(path, line, lineno, "missing space between %r and '('" % kw) + # eof + if eof and not line.endswith('\n'): + warn(path, line, lineno, "no blank line at EOF") + + ss = s.strip() + if ss.startswith(("printf(", "printf (", )): + if not ss.endswith(("// NOQA", "// NOQA")): + warn(path, line, lineno, "printf() statement") + + +def process(path): + with open(path, 'rt') as f: + lines = f.readlines() + for idx, line in enumerate(lines): + check_line(path, line, idx, lines) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('paths', nargs='+', help='path(s) to a file(s)') + args = parser.parse_args() + for path in args.paths: + process(path) + if warned: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/convert_readme.py b/scripts/internal/convert_readme.py new file mode 100755 index 00000000..d6cae918 --- /dev/null +++ b/scripts/internal/convert_readme.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Convert README.rst format to make it compatible with PyPI (no raw html). +""" + +import re +import sys + +summary = """\ +Quick links +=========== + +- `Home page <https://github.com/giampaolo/psutil>`_ +- `Install <https://github.com/giampaolo/psutil/blob/master/INSTALL.rst>`_ +- `Documentation <http://psutil.readthedocs.io>`_ +- `Download <https://pypi.org/project/psutil/#files>`_ +- `Forum <http://groups.google.com/group/psutil/topics>`_ +- `StackOverflow <https://stackoverflow.com/questions/tagged/psutil>`_ +- `Blog <https://gmpy.dev/tags/psutil>`_ +- `What's new <https://github.com/giampaolo/psutil/blob/master/HISTORY.rst>`_ +""" + +funding = """\ +Sponsors +======== + +.. image:: https://github.com/giampaolo/psutil/raw/master/docs/_static/tidelift-logo.png + :width: 200 + :alt: Alternative text + +`Add your logo <https://github.com/sponsors/giampaolo>`__. + +Example usages""" + + +def main(): + with open(sys.argv[1]) as f: + data = f.read() + data = re.sub(r".. raw:: html\n+\s+<div align[\s\S]*?/div>", summary, data) + data = re.sub(r"Sponsors\n========[\s\S]*?Example usages", funding, data) + print(data) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_wheels_appveyor.py index 4a559bb0..bc6c9717 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_wheels_appveyor.py @@ -1,11 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ -Script which downloads exe and wheel files hosted on AppVeyor: +Script which downloads wheel files hosted on AppVeyor: https://ci.appveyor.com/project/giampaolo/psutil Readapted from the original recipe of Ibarra Corretge' <saghul@gmail.com>: @@ -13,48 +13,27 @@ http://code.saghul.net/index.php/2015/09/09/ """ from __future__ import print_function -import argparse import concurrent.futures -import errno import os import requests -import shutil +import sys from psutil import __version__ as PSUTIL_VERSION from psutil._common import bytes2human -from scriptutils import printerr, exit +from psutil._common import print_color +USER = "giampaolo" +PROJECT = "psutil" BASE_URL = 'https://ci.appveyor.com/api' -PY_VERSIONS = ['2.7', '3.5', '3.6', '3.7', '3.8'] +PY_VERSIONS = ['2.7', '3.6', '3.7', '3.8', '3.9'] TIMEOUT = 30 -COLORS = True - - -def safe_makedirs(path): - try: - os.makedirs(path) - except OSError as err: - if err.errno == errno.EEXIST: - if not os.path.isdir(path): - raise - else: - raise - - -def safe_rmtree(path): - def onerror(fun, path, excinfo): - exc = excinfo[1] - if exc.errno != errno.ENOENT: - raise - - shutil.rmtree(path, onerror=onerror) def download_file(url): local_fname = url.split('/')[-1] local_fname = os.path.join('dist', local_fname) - safe_makedirs('dist') + os.makedirs('dist', exist_ok=True) r = requests.get(url, stream=True, timeout=TIMEOUT) tot_bytes = 0 with open(local_fname, 'wb') as f: @@ -65,10 +44,10 @@ def download_file(url): return local_fname -def get_file_urls(options): +def get_file_urls(): with requests.Session() as session: data = session.get( - BASE_URL + '/projects/' + options.user + '/' + options.project, + BASE_URL + '/projects/' + USER + '/' + PROJECT, timeout=TIMEOUT) data = data.json() @@ -81,13 +60,14 @@ def get_file_urls(options): file_url = job_url + '/' + item['fileName'] urls.append(file_url) if not urls: - exit("no artifacts found") + print_color("no artifacts found", 'red') + sys.exit(1) else: for url in sorted(urls, key=lambda x: os.path.basename(x)): yield url -def rename_27_wheels(): +def rename_win27_wheels(): # See: https://github.com/giampaolo/psutil/issues/810 src = 'dist/psutil-%s-cp27-cp27m-win32.whl' % PSUTIL_VERSION dst = 'dist/psutil-%s-cp27-none-win32.whl' % PSUTIL_VERSION @@ -99,9 +79,8 @@ def rename_27_wheels(): os.rename(src, dst) -def run(options): - safe_rmtree('dist') - urls = get_file_urls(options) +def run(): + urls = get_file_urls() completed = 0 exc = None with concurrent.futures.ThreadPoolExecutor() as e: @@ -111,7 +90,7 @@ def run(options): try: local_fname = fut.result() except Exception: - printerr("error while downloading %s" % (url)) + print_color("error while downloading %s" % (url), 'red') raise else: completed += 1 @@ -123,16 +102,11 @@ def run(options): return exit("expected %s files, got %s" % (expected, completed)) if exc: return exit() - rename_27_wheels() + rename_win27_wheels() def main(): - parser = argparse.ArgumentParser( - description='AppVeyor artifact downloader') - parser.add_argument('--user', required=True) - parser.add_argument('--project', required=True) - args = parser.parse_args() - run(args) + run() if __name__ == '__main__': diff --git a/scripts/internal/download_wheels_github.py b/scripts/internal/download_wheels_github.py new file mode 100755 index 00000000..a344ec49 --- /dev/null +++ b/scripts/internal/download_wheels_github.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Script which downloads wheel files hosted on GitHub: +https://github.com/giampaolo/psutil/actions +It needs an access token string generated from personal GitHub profile: +https://github.com/settings/tokens +The token must be created with at least "public_repo" scope/rights. +If you lose it, just generate a new token. +REST API doc: +https://developer.github.com/v3/actions/artifacts/ +""" + +import argparse +import json +import os +import requests +import sys +import zipfile + +from psutil import __version__ as PSUTIL_VERSION +from psutil._common import bytes2human +from psutil.tests import safe_rmpath + + +USER = "giampaolo" +PROJECT = "psutil" +OUTFILE = "wheels-github.zip" +TOKEN = "" + + +def get_artifacts(): + base_url = "https://api.github.com/repos/%s/%s" % (USER, PROJECT) + url = base_url + "/actions/artifacts" + res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN}) + res.raise_for_status() + data = json.loads(res.content) + return data + + +def download_zip(url): + print("downloading: " + url) + res = requests.get(url=url, headers={"Authorization": "token %s" % TOKEN}) + res.raise_for_status() + totbytes = 0 + with open(OUTFILE, 'wb') as f: + for chunk in res.iter_content(chunk_size=16384): + f.write(chunk) + totbytes += len(chunk) + print("got %s, size %s)" % (OUTFILE, bytes2human(totbytes))) + + +def rename_win27_wheels(): + # See: https://github.com/giampaolo/psutil/issues/810 + src = 'dist/psutil-%s-cp27-cp27m-win32.whl' % PSUTIL_VERSION + dst = 'dist/psutil-%s-cp27-none-win32.whl' % PSUTIL_VERSION + if os.path.exists(src): + print("rename: %s\n %s" % (src, dst)) + os.rename(src, dst) + src = 'dist/psutil-%s-cp27-cp27m-win_amd64.whl' % PSUTIL_VERSION + dst = 'dist/psutil-%s-cp27-none-win_amd64.whl' % PSUTIL_VERSION + if os.path.exists(src): + print("rename: %s\n %s" % (src, dst)) + os.rename(src, dst) + + +def run(): + data = get_artifacts() + download_zip(data['artifacts'][0]['archive_download_url']) + os.makedirs('dist', exist_ok=True) + with zipfile.ZipFile(OUTFILE, 'r') as zf: + zf.extractall('dist') + rename_win27_wheels() + + +def main(): + global TOKEN + parser = argparse.ArgumentParser(description='GitHub wheels downloader') + parser.add_argument('--token') + parser.add_argument('--tokenfile') + args = parser.parse_args() + + if args.tokenfile: + with open(os.path.expanduser(args.tokenfile)) as f: + TOKEN = f.read().strip() + elif args.token: + TOKEN = args.token + else: + return sys.exit('specify --token or --tokenfile args') + + try: + run() + finally: + safe_rmpath(OUTFILE) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/fix_flake8.py b/scripts/internal/fix_flake8.py index 5aa84db5..7cde608b 100644..100755 --- a/scripts/internal/fix_flake8.py +++ b/scripts/internal/fix_flake8.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/scripts/internal/generate_manifest.py b/scripts/internal/generate_manifest.py index f6cf0eaa..a1ed6b38 100755 --- a/scripts/internal/generate_manifest.py +++ b/scripts/internal/generate_manifest.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -12,8 +12,8 @@ import os import subprocess -SKIP_EXTS = ('.png', '.jpg', '.jpeg') -SKIP_FILES = ('.travis.yml', 'appveyor.yml') +SKIP_EXTS = ('.png', '.jpg', '.jpeg', '.svg') +SKIP_FILES = ('appveyor.yml') SKIP_PREFIXES = ('.ci/', '.github/') diff --git a/scripts/internal/git_pre_commit.py b/scripts/internal/git_pre_commit.py new file mode 100755 index 00000000..2ec4303d --- /dev/null +++ b/scripts/internal/git_pre_commit.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +This gets executed on 'git commit' and rejects the commit in case the +submitted code does not pass validation. Validation is run only against +the files which were modified in the commit. Checks: + +- assert no space at EOLs +- assert not pdb.set_trace in code +- assert no bare except clause ("except:") in code +- assert "flake8" checks pass +- assert C linter checks pass +- abort if files were added/renamed/removed and MANIFEST.in was not updated + +Install this with "make install-git-hooks". +""" + +from __future__ import print_function +import os +import subprocess +import sys + + +PYTHON = sys.executable +PY3 = sys.version_info[0] == 3 +THIS_SCRIPT = os.path.realpath(__file__) + + +def term_supports_colors(): + try: + import curses + assert sys.stderr.isatty() + curses.setupterm() + assert curses.tigetnum("colors") > 0 + except Exception: + return False + return True + + +def hilite(s, ok=True, bold=False): + """Return an highlighted version of 'string'.""" + if not term_supports_colors(): + return s + attr = [] + if ok is None: # no color + pass + elif ok: # green + attr.append('32') + else: # red + attr.append('31') + if bold: + attr.append('1') + return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) + + +def exit(msg): + print(hilite("commit aborted: " + msg, ok=False), file=sys.stderr) + sys.exit(1) + + +def sh(cmd): + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + if stderr: + print(stderr, file=sys.stderr) + if stdout.endswith('\n'): + stdout = stdout[:-1] + return stdout + + +def open_text(path): + kw = {'encoding': 'utf8'} if PY3 else {} + return open(path, 'rt', **kw) + + +def git_commit_files(): + out = sh("git diff --cached --name-only") + py_files = [x for x in out.split('\n') if x.endswith('.py') and + os.path.exists(x)] + c_files = [x for x in out.split('\n') if x.endswith(('.c', '.h')) and + os.path.exists(x)] + new_rm_mv = sh("git diff --name-only --diff-filter=ADR --cached") + # XXX: we should escape spaces and possibly other amenities here + new_rm_mv = new_rm_mv.split() + return (py_files, c_files, new_rm_mv) + + +def main(): + py_files, c_files, new_rm_mv = git_commit_files() + # Check file content. + for path in py_files: + if os.path.realpath(path) == THIS_SCRIPT: + continue + with open_text(path) as f: + lines = f.readlines() + for lineno, line in enumerate(lines, 1): + # space at end of line + if line.endswith(' '): + print("%s:%s %r" % (path, lineno, line)) + return exit("space at end of line") + line = line.rstrip() + # pdb + if "pdb.set_trace" in line: + print("%s:%s %s" % (path, lineno, line)) + return exit("you forgot a pdb in your python code") + # bare except clause + if "except:" in line and not line.endswith("# NOQA"): + print("%s:%s %s" % (path, lineno, line)) + return exit("bare except clause") + + # Python linter + if py_files: + assert os.path.exists('.flake8') + # XXX: we should escape spaces and possibly other amenities here + cmd = "%s -m flake8 --config=.flake8 %s" % (PYTHON, " ".join(py_files)) + ret = subprocess.call(cmd, shell=True) + if ret != 0: + return exit("python code is not flake8 compliant") + # C linter + if c_files: + # XXX: we should escape spaces and possibly other amenities here + cmd = "%s scripts/internal/clinter.py %s" % (PYTHON, " ".join(c_files)) + ret = subprocess.call(cmd, shell=True) + if ret != 0: + return exit("C code didn't pass style check") + if new_rm_mv: + out = sh("%s scripts/internal/generate_manifest.py" % PYTHON) + with open_text('MANIFEST.in') as f: + if out.strip() != f.read().strip(): + exit("some files were added, deleted or renamed; " + "run 'make generate-manifest' and commit again") + + +main() diff --git a/scripts/internal/print_access_denied.py b/scripts/internal/print_access_denied.py index 2c757fd7..81d192f0 100644..100755 --- a/scripts/internal/print_access_denied.py +++ b/scripts/internal/print_access_denied.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -50,7 +50,7 @@ from collections import defaultdict import time import psutil -from scriptutils import hilite +from psutil._common import print_color def main(): @@ -75,13 +75,12 @@ def main(): # print templ = "%-20s %-5s %-9s %s" s = templ % ("API", "AD", "Percent", "Outcome") - print(hilite(s, ok=None, bold=True)) + print_color(s, color=None, bold=True) for methname, ads in sorted(d.items(), key=lambda x: (x[1], x[0])): perc = (ads / tot_procs) * 100 outcome = "SUCCESS" if not ads else "ACCESS DENIED" s = templ % (methname, ads, "%6.1f%%" % perc, outcome) - s = hilite(s, ok=not ads) - print(s) + print_color(s, "red" if ads else None) tot_perc = round((tot_ads / tot_calls) * 100, 1) print("-" * 50) print("Totals: access-denied=%s (%s%%), calls=%s, processes=%s, " diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index 7fbe74f3..c9948c1d 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -6,16 +6,22 @@ """ Prints release announce based on HISTORY.rst file content. +See: https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode """ import os import re +import subprocess +import sys from psutil import __version__ as PRJ_VERSION HERE = os.path.abspath(os.path.dirname(__file__)) -HISTORY = os.path.abspath(os.path.join(HERE, '../../HISTORY.rst')) +ROOT = os.path.realpath(os.path.join(HERE, '..', '..')) +HISTORY = os.path.join(ROOT, 'HISTORY.rst') +PRINT_HASHES_SCRIPT = os.path.join( + ROOT, 'scripts', 'internal', 'print_hashes.py') PRJ_NAME = 'psutil' PRJ_URL_HOME = 'https://github.com/giampaolo/psutil' @@ -56,9 +62,14 @@ Links - Documentation: {prj_urldoc} - What's new: {prj_urlwhatsnew} +Hashes +====== + +{hashes} + -- -Giampaolo - http://grodola.blogspot.com +Giampaolo - https://gmpy.dev/about """ @@ -100,6 +111,8 @@ def get_changes(): def main(): changes = get_changes() + hashes = subprocess.check_output( + [sys.executable, PRINT_HASHES_SCRIPT, 'dist/']).strip().decode() print(template.format( prj_name=PRJ_NAME, prj_version=PRJ_VERSION, @@ -108,6 +121,7 @@ def main(): prj_urldoc=PRJ_URL_DOC, prj_urlwhatsnew=PRJ_URL_WHATSNEW, changes=changes, + hashes=hashes, )) diff --git a/scripts/internal/print_api_speed.py b/scripts/internal/print_api_speed.py index a99293c4..e39a1baa 100644..100755 --- a/scripts/internal/print_api_speed.py +++ b/scripts/internal/print_api_speed.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -9,53 +9,45 @@ $ make print_api_speed SYSTEM APIS SECONDS ---------------------------------- -boot_time 0.000140 -cpu_count 0.000016 -cpu_count (cores) 0.000312 -cpu_freq 0.000811 -cpu_percent 0.000138 -cpu_stats 0.000165 -cpu_times 0.000140 +cpu_count 0.000014 +disk_usage 0.000027 +cpu_times 0.000037 +cpu_percent 0.000045 ... PROCESS APIS SECONDS ---------------------------------- -children 0.007246 -cmdline 0.000069 -connections 0.000072 -cpu_affinity 0.000012 -cpu_num 0.000035 -cpu_percent 0.000042 -cpu_times 0.000031 +create_time 0.000001 +nice 0.000005 +cwd 0.000011 +cpu_affinity 0.000011 +ionice 0.000013 +... """ from __future__ import print_function, division from timeit import default_timer as timer -import argparse import inspect import os import psutil -from scriptutils import hilite +from psutil._common import print_color -SORT_BY_TIME = False if psutil.POSIX else True -TOP_SLOWEST = 7 timings = [] templ = "%-25s %s" def print_timings(): - slower = [] - timings.sort(key=lambda x: x[1 if SORT_BY_TIME else 0]) - for x in sorted(timings, key=lambda x: x[1], reverse=1)[:TOP_SLOWEST]: - slower.append(x[0]) + timings.sort(key=lambda x: x[1]) + i = 0 while timings[:]: title, elapsed = timings.pop(0) s = templ % (title, "%f" % elapsed) - if title in slower: - s = hilite(s, ok=False) - print(s) + if i > len(timings) - 5: + print_color(s, color="red") + else: + print(s) def timecall(title, fun, *args, **kw): @@ -65,11 +57,7 @@ def timecall(title, fun, *args, **kw): timings.append((title, elapsed)) -def titlestr(s): - return hilite(s, ok=None, bold=True) - - -def run(): +def main(): # --- system public_apis = [] @@ -83,7 +71,7 @@ def run(): if name not in ignore: public_apis.append(name) - print(titlestr(templ % ("SYSTEM APIS", "SECONDS"))) + print_color(templ % ("SYSTEM APIS", "SECONDS"), color=None, bold=True) print("-" * 34) for name in public_apis: fun = getattr(psutil, name) @@ -99,7 +87,7 @@ def run(): # --- process print("") - print(titlestr(templ % ("PROCESS APIS", "SECONDS"))) + print_color(templ % ("PROCESS APIS", "SECONDS"), color=None, bold=True) print("-" * 34) ignore = ['send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', 'as_dict', 'parent', 'parents', 'memory_info_ex', 'oneshot', @@ -114,18 +102,5 @@ def run(): print_timings() -def main(): - global SORT_BY_TIME, TOP_SLOWEST - parser = argparse.ArgumentParser(description='Benchmark all API calls') - parser.add_argument('-t', '--time', required=False, default=SORT_BY_TIME, - action='store_true', help="sort by timings") - parser.add_argument('-s', '--slowest', required=False, default=TOP_SLOWEST, - help="highlight the top N slowest APIs") - args = parser.parse_args() - SORT_BY_TIME = bool(args.time) - TOP_SLOWEST = int(args.slowest) - run() - - if __name__ == '__main__': main() diff --git a/scripts/internal/print_downloads.py b/scripts/internal/print_downloads.py new file mode 100755 index 00000000..7e5c4631 --- /dev/null +++ b/scripts/internal/print_downloads.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Print PYPI statistics in MarkDown format. +Useful sites: +* https://pepy.tech/project/psutil +* https://pypistats.org/packages/psutil +* https://hugovk.github.io/top-pypi-packages/ +""" + +from __future__ import print_function +import json +import os +import subprocess +import sys + +import pypinfo # NOQA + +from psutil._common import memoize + + +AUTH_FILE = os.path.expanduser("~/.pypinfo.json") +PKGNAME = 'psutil' +DAYS = 30 +LIMIT = 100 +GITHUB_SCRIPT_URL = "https://github.com/giampaolo/psutil/blob/master/" \ + "scripts/internal/pypistats.py" +LAST_UPDATE = None +bytes_billed = 0 + + +# --- get + +@memoize +def sh(cmd): + assert os.path.exists(AUTH_FILE) + env = os.environ.copy() + env['GOOGLE_APPLICATION_CREDENTIALS'] = AUTH_FILE + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + assert not stderr, stderr + return stdout.strip() + + +@memoize +def query(cmd): + global bytes_billed + ret = json.loads(sh(cmd)) + bytes_billed += ret['query']['bytes_billed'] + return ret + + +def top_packages(): + global LAST_UPDATE + ret = query("pypinfo --all --json --days %s --limit %s '' project" % ( + DAYS, LIMIT)) + LAST_UPDATE = ret['last_update'] + return [(x['project'], x['download_count']) for x in ret['rows']] + + +def ranking(): + data = top_packages() + i = 1 + for name, downloads in data: + if name == PKGNAME: + return i + i += 1 + raise ValueError("can't find %s" % PKGNAME) + + +def downloads(): + data = top_packages() + for name, downloads in data: + if name == PKGNAME: + return downloads + raise ValueError("can't find %s" % PKGNAME) + + +def downloads_pyver(): + return query("pypinfo --json --days %s %s pyversion" % (DAYS, PKGNAME)) + + +def downloads_by_country(): + return query("pypinfo --json --days %s %s country" % (DAYS, PKGNAME)) + + +def downloads_by_system(): + return query("pypinfo --json --days %s %s system" % (DAYS, PKGNAME)) + + +def downloads_by_distro(): + return query("pypinfo --json --days %s %s distro" % (DAYS, PKGNAME)) + + +# --- print + + +templ = "| %-30s | %15s |" + + +def print_row(left, right): + if isinstance(right, int): + right = '{0:,}'.format(right) + print(templ % (left, right)) + + +def print_header(left, right="Downloads"): + print_row(left, right) + s = templ % ("-" * 30, "-" * 15) + print("|:" + s[2:-2] + ":|") + + +def print_markdown_table(title, left, rows): + pleft = left.replace('_', ' ').capitalize() + print("### " + title) + print() + print_header(pleft) + for row in rows: + lval = row[left] + print_row(lval, row['download_count']) + print() + + +def main(): + downs = downloads() + + print("# Download stats") + print("") + s = "psutil download statistics of the last %s days (last update " % DAYS + s += "*%s*).\n" % LAST_UPDATE + s += "Generated via [pypistats.py](%s) script.\n" % GITHUB_SCRIPT_URL + print(s) + + data = [ + {'what': 'Per month', 'download_count': downs}, + {'what': 'Per day', 'download_count': int(downs / 30)}, + {'what': 'PYPI ranking', 'download_count': ranking()} + ] + print_markdown_table('Overview', 'what', data) + print_markdown_table('Operating systems', 'system_name', + downloads_by_system()['rows']) + print_markdown_table('Distros', 'distro_name', + downloads_by_distro()['rows']) + print_markdown_table('Python versions', 'python_version', + downloads_pyver()['rows']) + print_markdown_table('Countries', 'country', + downloads_by_country()['rows']) + + +if __name__ == '__main__': + try: + main() + finally: + print("bytes billed: %s" % bytes_billed, file=sys.stderr) diff --git a/scripts/internal/print_hashes.py b/scripts/internal/print_hashes.py new file mode 100755 index 00000000..434c43d5 --- /dev/null +++ b/scripts/internal/print_hashes.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Prints files hashes. +See: https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode +""" + +import hashlib +import os +import sys + + +def csum(file, kind): + h = hashlib.new(kind) + with open(file, "rb") as f: + h.update(f.read()) + return h.hexdigest() + + +def main(): + dir = sys.argv[1] + for name in sorted(os.listdir(dir)): + file = os.path.join(dir, name) + md5 = csum(file, "md5") + sha256 = csum(file, "sha256") + print("%s\nmd5: %s\nsha256: %s\n" % ( + os.path.basename(file), md5, sha256)) + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/print_timeline.py b/scripts/internal/print_timeline.py index 9bf6e9d1..64608b26 100644..100755 --- a/scripts/internal/print_timeline.py +++ b/scripts/internal/print_timeline.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/scripts/internal/print_wheels.py b/scripts/internal/print_wheels.py new file mode 100755 index 00000000..c2b8d36b --- /dev/null +++ b/scripts/internal/print_wheels.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Nicely print wheels print in dist/ directory.""" + +import collections +import glob +import os + +from psutil._common import print_color +from psutil._common import bytes2human + + +class Wheel: + + def __init__(self, path): + self._path = path + self._name = os.path.basename(path) + + def __repr__(self): + return "<Wheel(name=%s, plat=%s, arch=%s, pyver=%s)>" % ( + self.name, self.platform(), self.arch(), self.pyver()) + + __str__ = __repr__ + + @property + def name(self): + return self._name + + def platform(self): + plat = self.name.split('-')[-1] + pyimpl = self.name.split('-')[3] + ispypy = 'pypy' in pyimpl + if 'linux' in plat: + if ispypy: + return 'pypy_on_linux' + else: + return 'linux' + elif 'win' in plat: + if ispypy: + return 'pypy_on_windows' + else: + return 'windows' + elif 'macosx' in plat: + if ispypy: + return 'pypy_on_macos' + else: + return 'macos' + else: + raise ValueError("unknown platform %r" % self.name) + + def arch(self): + if self.name.endswith(('x86_64.whl', 'amd64.whl')): + return '64' + return '32' + + def pyver(self): + pyver = 'pypy' if self.name.split('-')[3].startswith('pypy') else 'py' + pyver += self.name.split('-')[2][2:] + return pyver + + def size(self): + return os.path.getsize(self._path) + + +def main(): + groups = collections.defaultdict(list) + for path in glob.glob('dist/*.whl'): + wheel = Wheel(path) + groups[wheel.platform()].append(wheel) + + tot_files = 0 + tot_size = 0 + templ = "%-54s %7s %7s %7s" + for platf, wheels in groups.items(): + ppn = "%s (total = %s)" % (platf, len(wheels)) + s = templ % (ppn, "size", "arch", "pyver") + print_color('\n' + s, color=None, bold=True) + for wheel in sorted(wheels, key=lambda x: x.name): + tot_files += 1 + tot_size += wheel.size() + s = templ % (wheel.name, bytes2human(wheel.size()), wheel.arch(), + wheel.pyver()) + if 'pypy' in wheel.pyver(): + print_color(s, color='violet') + else: + print_color(s, color='brown') + + print_color("\ntotals: files=%s, size=%s" % ( + tot_files, bytes2human(tot_size)), bold=True) + + +if __name__ == '__main__': + main() diff --git a/scripts/internal/purge_installation.py b/scripts/internal/purge_installation.py index d9301719..8a9597f0 100755 --- a/scripts/internal/purge_installation.py +++ b/scripts/internal/purge_installation.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -30,7 +30,7 @@ def rmpath(path): def main(): locations = [site.getusersitepackages()] - locations.extend(site.getsitepackages()) + locations += site.getsitepackages() for root in locations: if os.path.isdir(root): for name in os.listdir(root): diff --git a/scripts/internal/scriptutils.py b/scripts/internal/scriptutils.py deleted file mode 100644 index 3ee416f8..00000000 --- a/scripts/internal/scriptutils.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Utils shared by all files in scripts/internal.""" - -from __future__ import print_function -import sys - -from psutil._compat import lru_cache - - -__all__ = ['hilite', 'printerr', 'exit'] - - -@lru_cache() -def _term_supports_colors(file=sys.stdout): - try: - import curses - assert file.isatty() - curses.setupterm() - assert curses.tigetnum("colors") > 0 - except Exception: - return False - else: - return True - - -def hilite(s, ok=True, bold=False): - """Return an highlighted version of 'string'.""" - if not _term_supports_colors(): - return s - attr = [] - if ok is None: # no color - pass - elif ok: # green - attr.append('32') - else: # red - attr.append('31') - if bold: - attr.append('1') - return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) - - -def printerr(s): - print(hilite(s, ok=False), file=sys.stderr) - - -def exit(msg=""): - if msg: - printerr(msg) - sys.exit(1) diff --git a/scripts/internal/tidelift.py b/scripts/internal/tidelift.py new file mode 100755 index 00000000..fcba3e61 --- /dev/null +++ b/scripts/internal/tidelift.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Update news entry of Tidelift with latest HISTORY.rst section. +Put your Tidelift API token in a file first: +~/.tidelift.token +""" + +from __future__ import print_function +import os +import requests +import psutil +from psutil.tests import import_module_by_path + + +def upload_relnotes(package, version, text, token): + url = "https://api.tidelift.com/external-api/" + \ + "lifting/pypi/%s/release-notes/%s" % (package, version) + res = requests.put( + url=url, + data=text.encode('utf8'), + headers={"Authorization": "Bearer: %s" % token}) + print(version, res.status_code, res.text) + res.raise_for_status() + + +def main(): + here = os.path.abspath(os.path.dirname(__file__)) + path = os.path.join(here, "print_announce.py") + get_changes = import_module_by_path(path).get_changes + with open(os.path.expanduser("~/.tidelift.token")) as f: + token = f.read().strip() + upload_relnotes('psutil', psutil.__version__, get_changes(), token) + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index fe0a73dc..933951a2 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -31,7 +31,7 @@ if APPVEYOR: PYTHON = sys.executable else: PYTHON = os.getenv('PYTHON', sys.executable) -TEST_SCRIPT = 'psutil\\tests\\__main__.py' +RUNNER_PY = 'psutil\\tests\\runner.py' GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" PY3 = sys.version_info[0] == 3 HERE = os.path.abspath(os.path.dirname(__file__)) @@ -47,21 +47,19 @@ DEPS = [ "pyreadline", "setuptools", "wheel", - "wmi", "requests" ] -if sys.version_info[:2] <= (2, 6): +if sys.version_info[:2] <= (2, 7): DEPS.append('unittest2') if sys.version_info[:2] <= (2, 7): DEPS.append('mock') if sys.version_info[:2] <= (3, 2): DEPS.append('ipaddress') -if PYPY: - pass -elif sys.version_info[:2] <= (3, 4): - DEPS.append("pypiwin32==219") -else: - DEPS.append("pypiwin32") +if sys.version_info[:2] <= (3, 4): + DEPS.append('enum34') +if not PYPY: + DEPS.append("pywin32") + DEPS.append("wmi") _cmds = {} if PY3: @@ -210,12 +208,6 @@ def recursive_rm(*patterns): safe_rmtree(os.path.join(root, dir)) -def test_setup(): - os.environ['PYTHONWARNINGS'] = 'all' - os.environ['PSUTIL_TESTING'] = '1' - os.environ['PSUTIL_DEBUG'] = '1' - - # =================================================================== # commands # =================================================================== @@ -227,8 +219,13 @@ def build(): # edit mode). sh('%s -c "import setuptools"' % PYTHON) + # "build_ext -i" copies compiled *.pyd files in ./psutil directory in + # order to allow "import psutil" when using the interactive interpreter + # from within psutil root directory. + cmd = [PYTHON, "setup.py", "build_ext", "-i"] + if sys.version_info[:2] >= (3, 6) and os.cpu_count() or 1 > 1: + cmd += ['--parallel', str(os.cpu_count())] # Print coloured warnings in real time. - cmd = [PYTHON, "setup.py", "build"] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) try: for line in iter(p.stdout.readline, b''): @@ -250,10 +247,6 @@ def build(): p.terminate() p.wait() - # Copies compiled *.pyd files in ./psutil directory in order to - # allow "import psutil" when using the interactive interpreter - # from within this directory. - sh("%s setup.py build_ext -i" % PYTHON) # Make sure it actually worked. sh('%s -c "import psutil"' % PYTHON) win_colorprint("build + import successful", GREEN) @@ -369,7 +362,6 @@ def clean(): "*__pycache__", ".coverage", ".failed-tests.txt", - ".tox", ) safe_rmtree("build") safe_rmtree(".coverage") @@ -386,7 +378,7 @@ def setup_dev_env(): sh("%s -m pip install -U %s" % (PYTHON, " ".join(DEPS))) -def flake8(): +def lint(): """Run flake8 against all py files""" py_files = subprocess.check_output("git ls-files") if PY3: @@ -396,21 +388,17 @@ def flake8(): sh("%s -m flake8 %s" % (PYTHON, py_files), nolog=True) -def test(script=TEST_SCRIPT): +def test(name=RUNNER_PY): """Run tests""" - install() - test_setup() - cmdline = "%s %s" % (PYTHON, script) - safe_print(cmdline) - sh(cmdline) + build() + sh("%s %s" % (PYTHON, name)) def coverage(): """Run coverage tests.""" # Note: coverage options are controlled by .coveragerc file - install() - test_setup() - sh("%s -m coverage run %s" % (PYTHON, TEST_SCRIPT)) + build() + sh("%s -m coverage run %s" % (PYTHON, RUNNER_PY)) sh("%s -m coverage report" % PYTHON) sh("%s -m coverage html" % PYTHON) sh("%s -m webbrowser -t htmlcov/index.html" % PYTHON) @@ -418,79 +406,75 @@ def coverage(): def test_process(): """Run process tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_process.py" % PYTHON) def test_system(): """Run system tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_system.py" % PYTHON) def test_platform(): """Run windows only tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_windows.py" % PYTHON) def test_misc(): """Run misc tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_misc.py" % PYTHON) def test_unicode(): """Run unicode tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_unicode.py" % PYTHON) def test_connections(): """Run connections tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_connections.py" % PYTHON) def test_contracts(): """Run contracts tests""" - install() - test_setup() + build() sh("%s psutil\\tests\\test_contracts.py" % PYTHON) +def test_testutils(): + """Run test utilities tests""" + build() + sh("%s psutil\\tests\\test_testutils.py" % PYTHON) + + def test_by_name(name): """Run test by name""" - install() - test_setup() + build() sh("%s -m unittest -v %s" % (PYTHON, name)) def test_failed(): """Re-run tests which failed on last run.""" - install() - test_setup() - sh('%s -c "import psutil.tests.runner as r; r.run(last_failed=True)"' % ( - PYTHON)) + build() + sh("%s %s --last-failed" % (PYTHON, RUNNER_PY)) def test_memleaks(): """Run memory leaks tests""" - install() - test_setup() - sh("%s psutil\\tests\\test_memory_leaks.py" % PYTHON) + build() + sh("%s psutil\\tests\\test_memleaks.py" % PYTHON) def install_git_hooks(): """Install GIT pre-commit hook.""" if os.path.isdir('.git'): - src = os.path.join(ROOT_DIR, "scripts", "internal", ".git-pre-commit") + src = os.path.join( + ROOT_DIR, "scripts", "internal", "git_pre_commit.py") dst = os.path.realpath( os.path.join(ROOT_DIR, ".git", "hooks", "pre-commit")) with open(src, "rt") as s: @@ -510,18 +494,22 @@ def bench_oneshot_2(): def print_access_denied(): """Print AD exceptions raised by all Process methods.""" - install() - test_setup() + build() sh("%s -Wa scripts\\internal\\print_access_denied.py" % PYTHON) def print_api_speed(): """Benchmark all API calls.""" - install() - test_setup() + build() sh("%s -Wa scripts\\internal\\print_api_speed.py" % PYTHON) +def download_appveyor_wheels(): + """Download appveyor wheels.""" + sh("%s -Wa scripts\\internal\\download_wheels_appveyor.py " + "--user giampaolo --project psutil" % PYTHON) + + def get_python(path): if not path: return sys.executable @@ -529,9 +517,25 @@ def get_python(path): return path # try to look for a python installation given a shortcut name path = path.replace('.', '') - vers = ('26', '27', '36', '37', '38', - '26-64', '27-64', '36-64', '37-64', '38-64' - '26-32', '27-32', '36-32', '37-32', '38-32') + vers = ( + '26', + '26-32', + '26-64', + '27', + '27-32', + '27-64', + '36', + '36-32', + '36-64', + '37', + '37-32', + '37-64', + '38', + '38-32', + '38-64', + '39-32', + '39-64', + ) for v in vers: pypath = r'C:\\python%s\python.exe' % v if path in pypath and os.path.isfile(pypath): @@ -551,11 +555,12 @@ def main(): sp.add_parser('build', help="build") sp.add_parser('clean', help="deletes dev files") sp.add_parser('coverage', help="run coverage tests.") - sp.add_parser('flake8', help="run flake8 against all py files") + sp.add_parser('download-appveyor-wheels', help="download wheels.") sp.add_parser('help', help="print this help") sp.add_parser('install', help="build + install in develop/edit mode") sp.add_parser('install-git-hooks', help="install GIT pre-commit hook") sp.add_parser('install-pip', help="install pip") + sp.add_parser('lint', help="run flake8 against all py files") sp.add_parser('print-access-denied', help="print AD exceptions") sp.add_parser('print-api-speed', help="benchmark all API calls") sp.add_parser('setup-dev-env', help="install deps") @@ -570,6 +575,7 @@ def main(): sp.add_parser('test-process', help="run process tests") sp.add_parser('test-system', help="run system tests") sp.add_parser('test-unicode', help="run unicode tests") + sp.add_parser('test-testutils', help="run test utils tests") sp.add_parser('uninstall', help="uninstall psutil") sp.add_parser('upload-wheels', help="upload wheel files on PyPI") sp.add_parser('wheel', help="create wheel file") diff --git a/scripts/iotop.py b/scripts/iotop.py index 6a5d3fa9..04683673 100755 --- a/scripts/iotop.py +++ b/scripts/iotop.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -14,7 +14,7 @@ It doesn't work on Windows as curses module is required. Example output: -$ python scripts/iotop.py +$ python3 scripts/iotop.py Total DISK READ: 0.00 B/s | Total DISK WRITE: 472.00 K/s PID USER DISK READ DISK WRITE COMMAND 13155 giampao 0.00 B/s 428.00 K/s /usr/bin/google-chrome-beta @@ -30,7 +30,6 @@ PID USER DISK READ DISK WRITE COMMAND Author: Giampaolo Rodola' <g.rodola@gmail.com> """ -import atexit import time import sys try: @@ -42,20 +41,11 @@ import psutil from psutil._common import bytes2human -def tear_down(): - win.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - - win = curses.initscr() -atexit.register(tear_down) -curses.endwin() lineno = 0 -def print_line(line, highlight=False): +def printl(line, highlight=False): """A thin wrapper around curses's addstr().""" global lineno try: @@ -129,10 +119,10 @@ def refresh_window(procs, disks_read, disks_write): disks_tot = "Total DISK READ: %s | Total DISK WRITE: %s" \ % (bytes2human(disks_read), bytes2human(disks_write)) - print_line(disks_tot) + printl(disks_tot) header = templ % ("PID", "USER", "DISK READ", "DISK WRITE", "COMMAND") - print_line(header, highlight=True) + printl(header, highlight=True) for p in procs: line = templ % ( @@ -142,21 +132,45 @@ def refresh_window(procs, disks_read, disks_write): bytes2human(p._write_per_sec), p._cmdline) try: - print_line(line) + printl(line) except curses.error: break win.refresh() +def setup(): + curses.start_color() + curses.use_default_colors() + for i in range(0, curses.COLORS): + curses.init_pair(i + 1, i, -1) + curses.endwin() + win.nodelay(1) + + +def tear_down(): + win.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + def main(): + global lineno + setup() try: interval = 0 while True: + if win.getch() == ord('q'): + break args = poll(interval) refresh_window(*args) - interval = 1 + lineno = 0 + interval = 0.5 + time.sleep(interval) except (KeyboardInterrupt, SystemExit): pass + finally: + tear_down() if __name__ == '__main__': diff --git a/scripts/killall.py b/scripts/killall.py index f9cc9201..7bbcd75a 100755 --- a/scripts/killall.py +++ b/scripts/killall.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be diff --git a/scripts/meminfo.py b/scripts/meminfo.py index 0b15ff2b..b98aa60b 100755 --- a/scripts/meminfo.py +++ b/scripts/meminfo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ Print system memory information. -$ python scripts/meminfo.py +$ python3 scripts/meminfo.py MEMORY ------ Total : 9.7G diff --git a/scripts/netstat.py b/scripts/netstat.py index 490b429f..1832a096 100755 --- a/scripts/netstat.py +++ b/scripts/netstat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ A clone of 'netstat -antp' on Linux. -$ python scripts/netstat.py +$ python3 scripts/netstat.py Proto Local address Remote address Status PID Program name tcp 127.0.0.1:48256 127.0.0.1:45884 ESTABLISHED 13646 chrome tcp 127.0.0.1:47073 127.0.0.1:45884 ESTABLISHED 13646 chrome @@ -41,20 +41,21 @@ def main(): "Proto", "Local address", "Remote address", "Status", "PID", "Program name")) proc_names = {} - for p in psutil.process_iter(attrs=['pid', 'name']): + for p in psutil.process_iter(['pid', 'name']): proc_names[p.info['pid']] = p.info['name'] for c in psutil.net_connections(kind='inet'): laddr = "%s:%s" % (c.laddr) raddr = "" if c.raddr: raddr = "%s:%s" % (c.raddr) + name = proc_names.get(c.pid, '?') or '' print(templ % ( proto_map[(c.family, c.type)], laddr, raddr or AD, c.status, c.pid or AD, - proc_names.get(c.pid, '?')[:15], + name[:15], )) diff --git a/scripts/nettop.py b/scripts/nettop.py index 45b8879f..8cc19fda 100755 --- a/scripts/nettop.py +++ b/scripts/nettop.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # $Id: iotop.py 1160 2011-10-14 18:50:36Z g.rodola@gmail.com $ # @@ -11,7 +11,7 @@ Shows real-time network statistics. Author: Giampaolo Rodola' <g.rodola@gmail.com> -$ python scripts/nettop.py +$ python3 scripts/nettop.py ----------------------------------------------------------- total bytes: sent: 1.49 G received: 4.82 G total packets: sent: 7338724 received: 8082712 @@ -31,7 +31,6 @@ pkts-sent 0 0 pkts-recv 1214470 0 """ -import atexit import time import sys try: @@ -43,20 +42,11 @@ import psutil from psutil._common import bytes2human -def tear_down(): - win.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - - -win = curses.initscr() -atexit.register(tear_down) -curses.endwin() lineno = 0 +win = curses.initscr() -def print_line(line, highlight=False): +def printl(line, highlight=False): """A thin wrapper around curses's addstr().""" global lineno try: @@ -89,59 +79,80 @@ def refresh_window(tot_before, tot_after, pnic_before, pnic_after): global lineno # totals - print_line("total bytes: sent: %-10s received: %s" % ( + printl("total bytes: sent: %-10s received: %s" % ( bytes2human(tot_after.bytes_sent), bytes2human(tot_after.bytes_recv)) ) - print_line("total packets: sent: %-10s received: %s" % ( + printl("total packets: sent: %-10s received: %s" % ( tot_after.packets_sent, tot_after.packets_recv)) # per-network interface details: let's sort network interfaces so # that the ones which generated more traffic are shown first - print_line("") + printl("") nic_names = list(pnic_after.keys()) nic_names.sort(key=lambda x: sum(pnic_after[x]), reverse=True) for name in nic_names: stats_before = pnic_before[name] stats_after = pnic_after[name] templ = "%-15s %15s %15s" - print_line(templ % (name, "TOTAL", "PER-SEC"), highlight=True) - print_line(templ % ( + printl(templ % (name, "TOTAL", "PER-SEC"), highlight=True) + printl(templ % ( "bytes-sent", bytes2human(stats_after.bytes_sent), bytes2human( stats_after.bytes_sent - stats_before.bytes_sent) + '/s', )) - print_line(templ % ( + printl(templ % ( "bytes-recv", bytes2human(stats_after.bytes_recv), bytes2human( stats_after.bytes_recv - stats_before.bytes_recv) + '/s', )) - print_line(templ % ( + printl(templ % ( "pkts-sent", stats_after.packets_sent, stats_after.packets_sent - stats_before.packets_sent, )) - print_line(templ % ( + printl(templ % ( "pkts-recv", stats_after.packets_recv, stats_after.packets_recv - stats_before.packets_recv, )) - print_line("") + printl("") win.refresh() lineno = 0 +def setup(): + curses.start_color() + curses.use_default_colors() + for i in range(0, curses.COLORS): + curses.init_pair(i + 1, i, -1) + curses.endwin() + win.nodelay(1) + + +def tear_down(): + win.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + def main(): + setup() try: interval = 0 while True: + if win.getch() == ord('q'): + break args = poll(interval) refresh_window(*args) - interval = 1 + interval = 0.5 except (KeyboardInterrupt, SystemExit): pass + finally: + tear_down() if __name__ == '__main__': diff --git a/scripts/pidof.py b/scripts/pidof.py index bcb8a2e6..ee18aae4 100755 --- a/scripts/pidof.py +++ b/scripts/pidof.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola', karthikrev. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -18,7 +18,7 @@ import sys def pidof(pgname): pids = [] - for proc in psutil.process_iter(attrs=['name', 'cmdline']): + for proc in psutil.process_iter(['name', 'cmdline']): # search for matches in the process name and cmdline if proc.info['name'] == pgname or \ proc.info['cmdline'] and proc.info['cmdline'][0] == pgname: diff --git a/scripts/pmap.py b/scripts/pmap.py index 300f23e9..459927bf 100755 --- a/scripts/pmap.py +++ b/scripts/pmap.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,7 +8,7 @@ A clone of 'pmap' utility on Linux, 'vmmap' on macOS and 'procstat -v' on BSD. Report memory map of a process. -$ python scripts/pmap.py 32402 +$ python3 scripts/pmap.py 32402 Address RSS Mode Mapping 0000000000400000 1200K r-xp /usr/bin/python2.7 0000000000838000 4K r--p /usr/bin/python2.7 diff --git a/scripts/procinfo.py b/scripts/procinfo.py index 161b5057..743a1777 100755 --- a/scripts/procinfo.py +++ b/scripts/procinfo.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,7 +8,7 @@ Print detailed information about a process. Author: Giampaolo Rodola' <g.rodola@gmail.com> -$ python scripts/procinfo.py +$ python3 scripts/procinfo.py pid 4600 name chrome parent 4554 (bash) @@ -102,17 +102,19 @@ RLIMITS_MAP = { "RLIMIT_CPU": "cputime", "RLIMIT_DATA": "datasize", "RLIMIT_FSIZE": "filesize", - "RLIMIT_LOCKS": "locks", "RLIMIT_MEMLOCK": "memlock", "RLIMIT_MSGQUEUE": "msgqueue", "RLIMIT_NICE": "nice", "RLIMIT_NOFILE": "openfiles", "RLIMIT_NPROC": "maxprocesses", + "RLIMIT_NPTS": "pseudoterms", "RLIMIT_RSS": "rss", "RLIMIT_RTPRIO": "realtimeprio", "RLIMIT_RTTIME": "rtimesched", + "RLIMIT_SBSIZE": "sockbufsize", "RLIMIT_SIGPENDING": "sigspending", "RLIMIT_STACK": "stack", + "RLIMIT_SWAP": "swapuse", } @@ -317,7 +319,7 @@ def run(pid, verbose=False): def main(argv=None): parser = argparse.ArgumentParser( description="print information about a process") - parser.add_argument("pid", type=int, help="process pid") + parser.add_argument("pid", type=int, help="process pid", nargs='?') parser.add_argument('--verbose', '-v', action='store_true', help="print more info") args = parser.parse_args() diff --git a/scripts/procsmem.py b/scripts/procsmem.py index f660c085..1074c4c2 100755 --- a/scripts/procsmem.py +++ b/scripts/procsmem.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -64,7 +64,7 @@ def main(): with p.oneshot(): try: mem = p.memory_full_info() - info = p.as_dict(attrs=["cmdline", "username"]) + info = p.as_dict(["cmdline", "username"]) except psutil.AccessDenied: ad_pids.append(p.pid) except psutil.NoSuchProcess: @@ -80,18 +80,19 @@ def main(): procs.append(p) procs.sort(key=lambda p: p._uss) - templ = "%-7s %-7s %-30s %7s %7s %7s %7s" - print(templ % ("PID", "User", "Cmdline", "USS", "PSS", "Swap", "RSS")) + templ = "%-7s %-7s %7s %7s %7s %7s %7s" + print(templ % ("PID", "User", "USS", "PSS", "Swap", "RSS", "Cmdline")) print("=" * 78) for p in procs[:86]: + cmd = " ".join(p._info["cmdline"])[:50] if p._info["cmdline"] else "" line = templ % ( p.pid, p._info["username"][:7] if p._info["username"] else "", - " ".join(p._info["cmdline"])[:30], convert_bytes(p._uss), convert_bytes(p._pss) if p._pss != "" else "", convert_bytes(p._swap) if p._swap != "" else "", convert_bytes(p._rss), + cmd, ) print(line) if ad_pids: diff --git a/scripts/ps.py b/scripts/ps.py index 8467cca6..a234209f 100755 --- a/scripts/ps.py +++ b/scripts/ps.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ """ A clone of 'ps aux'. -$ python scripts/ps.py +$ python3 scripts/ps.py USER PID %MEM VSZ RSS NICE STATUS START TIME CMDLINE root 1 0.0 220.9M 6.5M sleep Mar27 09:10 /lib/systemd root 2 0.0 0.0B 0.0B sleep Mar27 00:00 kthreadd diff --git a/scripts/pstree.py b/scripts/pstree.py index 8e4c9f95..dba9f1bd 100755 --- a/scripts/pstree.py +++ b/scripts/pstree.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,7 +8,7 @@ Similar to 'ps aux --forest' on Linux, prints the process list as a tree structure. -$ python scripts/pstree.py +$ python3 scripts/pstree.py 0 ? |- 1 init | |- 289 cgmanager diff --git a/scripts/sensors.py b/scripts/sensors.py index bbf3ac90..911d7c9b 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. @@ -9,7 +9,7 @@ A clone of 'sensors' utility on Linux printing hardware temperatures, fans speed and battery info. -$ python scripts/sensors.py +$ python3 scripts/sensors.py asus Temperatures: asus 57.0°C (high=None°C, critical=None°C) @@ -30,7 +30,6 @@ Battery: """ from __future__ import print_function - import psutil diff --git a/scripts/temperatures.py b/scripts/temperatures.py index 15b9156b..f2dd51a7 100755 --- a/scripts/temperatures.py +++ b/scripts/temperatures.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. @@ -8,7 +8,7 @@ """ A clone of 'sensors' utility on Linux printing hardware temperatures. -$ python scripts/sensors.py +$ python3 scripts/sensors.py asus asus 47.0 °C (high = None °C, critical = None °C) diff --git a/scripts/top.py b/scripts/top.py index 69890e27..989f8306 100755 --- a/scripts/top.py +++ b/scripts/top.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -9,7 +9,7 @@ A clone of top / htop. Author: Giampaolo Rodola' <g.rodola@gmail.com> -$ python scripts/top.py +$ python3 scripts/top.py CPU0 [|||| ] 10.9% CPU1 [||||| ] 13.1% CPU2 [||||| ] 12.8% @@ -33,7 +33,6 @@ PID USER NI VIRT RES CPU% MEM% TIME+ NAME ... """ -import atexit import datetime import sys import time @@ -46,30 +45,28 @@ import psutil from psutil._common import bytes2human -# --- curses stuff - -def tear_down(): - win.keypad(0) - curses.nocbreak() - curses.echo() - curses.endwin() - - win = curses.initscr() -atexit.register(tear_down) -curses.endwin() lineno = 0 +colors_map = dict( + green=3, + red=10, + yellow=4, +) -def print_line(line, highlight=False): +def printl(line, color=None, bold=False, highlight=False): """A thin wrapper around curses's addstr().""" global lineno try: + flags = 0 + if color: + flags |= curses.color_pair(colors_map[color]) + if bold: + flags |= curses.A_BOLD if highlight: line += " " * (win.getmaxyx()[1] - len(line)) - win.addstr(lineno, 0, line, curses.A_REVERSE) - else: - win.addstr(lineno, 0, line, 0) + flags |= curses.A_STANDOUT + win.addstr(lineno, 0, line, flags) except curses.error: lineno = 0 win.refresh() @@ -105,6 +102,15 @@ def poll(interval): return (processes, procs_status) +def get_color(perc): + if perc <= 30: + return "green" + elif perc <= 80: + return "yellow" + else: + return "red" + + def print_header(procs_status, num_procs): """Print system-related info, above the process list.""" @@ -117,17 +123,19 @@ def print_header(procs_status, num_procs): percs = psutil.cpu_percent(interval=0, percpu=True) for cpu_num, perc in enumerate(percs): dashes, empty_dashes = get_dashes(perc) - print_line(" CPU%-2s [%s%s] %5s%%" % (cpu_num, dashes, empty_dashes, - perc)) + line = " CPU%-2s [%s%s] %5s%%" % (cpu_num, dashes, empty_dashes, perc) + printl(line, color=get_color(perc)) + + # memory usage mem = psutil.virtual_memory() dashes, empty_dashes = get_dashes(mem.percent) line = " Mem [%s%s] %5s%% %6s / %s" % ( dashes, empty_dashes, mem.percent, - str(int(mem.used / 1024 / 1024)) + "M", - str(int(mem.total / 1024 / 1024)) + "M" + bytes2human(mem.used), + bytes2human(mem.total), ) - print_line(line) + printl(line, color=get_color(mem.percent)) # swap usage swap = psutil.swap_memory() @@ -135,10 +143,10 @@ def print_header(procs_status, num_procs): line = " Swap [%s%s] %5s%% %6s / %s" % ( dashes, empty_dashes, swap.percent, - str(int(swap.used / 1024 / 1024)) + "M", - str(int(swap.total / 1024 / 1024)) + "M" + bytes2human(swap.used), + bytes2human(swap.total), ) - print_line(line) + printl(line, color=get_color(swap.percent)) # processes number and status st = [] @@ -146,14 +154,14 @@ def print_header(procs_status, num_procs): if y: st.append("%s=%s" % (x, y)) st.sort(key=lambda x: x[:3] in ('run', 'sle'), reverse=1) - print_line(" Processes: %s (%s)" % (num_procs, ', '.join(st))) + printl(" Processes: %s (%s)" % (num_procs, ', '.join(st))) # load average, uptime uptime = datetime.datetime.now() - \ datetime.datetime.fromtimestamp(psutil.boot_time()) av1, av2, av3 = psutil.getloadavg() line = " Load average: %.2f %.2f %.2f Uptime: %s" \ % (av1, av2, av3, str(uptime).split('.')[0]) - print_line(line) + printl(line) def refresh_window(procs, procs_status): @@ -164,8 +172,8 @@ def refresh_window(procs, procs_status): header = templ % ("PID", "USER", "NI", "VIRT", "RES", "CPU%", "MEM%", "TIME+", "NAME") print_header(procs_status, len(procs)) - print_line("") - print_line(header, highlight=True) + printl("") + printl(header, bold=True, highlight=True) for p in procs: # TIME+ column shows process CPU cumulative time and it # is expressed as: "mm:ss.ms" @@ -197,21 +205,42 @@ def refresh_window(procs, procs_status): p.dict['name'] or '', ) try: - print_line(line) + printl(line) except curses.error: break win.refresh() +def setup(): + curses.start_color() + curses.use_default_colors() + for i in range(0, curses.COLORS): + curses.init_pair(i + 1, i, -1) + curses.endwin() + win.nodelay(1) + + +def tear_down(): + win.keypad(0) + curses.nocbreak() + curses.echo() + curses.endwin() + + def main(): + setup() try: interval = 0 while True: + if win.getch() == ord('q'): + break args = poll(interval) refresh_window(*args) interval = 1 except (KeyboardInterrupt, SystemExit): pass + finally: + tear_down() if __name__ == '__main__': diff --git a/scripts/who.py b/scripts/who.py index 748d936c..c1e40729 100755 --- a/scripts/who.py +++ b/scripts/who.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -8,7 +8,7 @@ A clone of 'who' command; print information about users who are currently logged in. -$ python scripts/who.py +$ python3 scripts/who.py giampaolo console 2017-03-25 22:24 loginwindow giampaolo ttys000 2017-03-25 23:28 (10.0.2.2) sshd """ diff --git a/scripts/winservices.py b/scripts/winservices.py index 67724835..5c710159 100755 --- a/scripts/winservices.py +++ b/scripts/winservices.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -7,7 +7,7 @@ r""" List all Windows services installed. -$ python scripts/winservices.py +$ python3 scripts/winservices.py AeLookupSvc (Application Experience) status: stopped, start: manual, username: localSystem, pid: None binpath: C:\Windows\system32\svchost.exe -k netsvcs @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -14,6 +14,7 @@ import platform import re import shutil import struct +import subprocess import sys import tempfile import warnings @@ -47,6 +48,7 @@ from _compat import PY3 # NOQA from _compat import which # NOQA +PYPY = '__pypy__' in sys.builtin_module_names macros = [] if POSIX: macros.append(("PSUTIL_POSIX", 1)) @@ -66,17 +68,17 @@ sources = ['psutil/_psutil_common.c'] if POSIX: sources.append('psutil/_psutil_posix.c') -tests_require = [] -if sys.version_info[:2] <= (2, 6): - tests_require.append('unittest2') -if sys.version_info[:2] <= (2, 7): - tests_require.append('mock') -if sys.version_info[:2] <= (3, 2): - tests_require.append('ipaddress') -extras_require = {} -if sys.version_info[:2] <= (3, 3): - extras_require.update(dict(enum='enum34')) +extras_require = {"test": [ + "enum34; python_version <= '3.4'", + "ipaddress; python_version < '3.0'", + "mock; python_version < '3.0'", + "unittest2; python_version < '3.0'", +]} +if not PYPY: + extras_require['test'].extend([ + "pywin32; sys.platform == 'win32'", + "wmi; sys.platform == 'win32'"]) def get_version(): @@ -97,9 +99,17 @@ macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) def get_description(): - README = os.path.join(HERE, 'README.rst') - with open(README, 'r') as f: - return f.read() + script = os.path.join(HERE, "scripts", "internal", "convert_readme.py") + readme = os.path.join(HERE, 'README.rst') + p = subprocess.Popen([sys.executable, script, readme], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + data = stdout.decode('utf8') + if WINDOWS: + data = data.replace('\r\n', '\n') + return data @contextlib.contextmanager @@ -120,9 +130,9 @@ def silenced_output(stream_name): def missdeps(msg): - s = hilite("C compiler or Python headers are not installed ", ok=False) - s += hilite("on this system. Try to run:\n", ok=False) - s += hilite(msg, ok=False, bold=True) + s = hilite("C compiler or Python headers are not installed ", color="red") + s += hilite("on this system. Try to run:\n", color="red") + s += hilite(msg, color="red", bold=True) print(s, file=sys.stderr) @@ -166,7 +176,7 @@ if WINDOWS: define_macros=macros, libraries=[ "psapi", "kernel32", "advapi32", "shell32", "netapi32", - "wtsapi32", "ws2_32", "PowrProf", "pdh", + "ws2_32", "PowrProf", "pdh", ], # extra_compile_args=["/W 4"], # extra_link_args=["/DEBUG"] @@ -179,6 +189,7 @@ elif MACOS: sources=sources + [ 'psutil/_psutil_osx.c', 'psutil/arch/osx/process_info.c', + 'psutil/arch/osx/cpu.c', ], define_macros=macros, extra_link_args=[ @@ -191,6 +202,7 @@ elif FREEBSD: 'psutil._psutil_bsd', sources=sources + [ 'psutil/_psutil_bsd.c', + 'psutil/arch/freebsd/cpu.c', 'psutil/arch/freebsd/specific.c', 'psutil/arch/freebsd/sys_socks.c', 'psutil/arch/freebsd/proc_socks.c', @@ -327,6 +339,7 @@ def main(): version=VERSION, description=__doc__ .replace('\n', ' ').strip() if __doc__ else '', long_description=get_description(), + long_description_content_type='text/x-rst', keywords=[ 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'nice', 'tty', 'ionice', 'uptime', 'taskmgr', 'process', 'df', 'iotop', 'iostat', @@ -385,7 +398,6 @@ def main(): 'Topic :: System :: Networking :: Monitoring', 'Topic :: System :: Networking', 'Topic :: System :: Operating System', - 'Topic :: System :: Power (UPS)' 'Topic :: System :: Systems Administration', 'Topic :: Utilities', ], @@ -393,8 +405,6 @@ def main(): if setuptools is not None: kwargs.update( python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", - test_suite="psutil.tests.get_suite", - tests_require=tests_require, extras_require=extras_require, zip_safe=False, ) @@ -412,7 +422,7 @@ def main(): missdeps("sudo yum install gcc python%s-devel" % py3) elif MACOS: print(hilite("XCode (https://developer.apple.com/xcode/) " - "is not installed"), ok=False, file=sys.stderr) + "is not installed"), color="red", file=sys.stderr) elif FREEBSD: missdeps("pkg install gcc python%s" % py3) elif OPENBSD: @@ -422,14 +432,6 @@ def main(): elif SUNOS: missdeps("sudo ln -s /usr/bin/gcc /usr/local/bin/cc && " "pkg install gcc") - elif not success and WINDOWS: - if PY3: - ur = "http://www.visualstudio.com/en-au/news/vs2015-preview-vs" - else: - ur = "http://www.microsoft.com/en-us/download/" - ur += "details.aspx?id=44266" - s = "VisualStudio is not installed; get it from %s" % ur - print(hilite(s, ok=False), file=sys.stderr) if __name__ == '__main__': diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 2698eb6a..00000000 --- a/tox.ini +++ /dev/null @@ -1,28 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. -# To use it run "pip install tox" and then run "tox" from this -# directory. - -[tox] -envlist = py26, py27, py34, py35, py36, py37, py38, lint - -[testenv] -deps = - py26: ipaddress - py26: mock==1.0.1 - py26: unittest2 - py27: ipaddress - py27: mock - -setenv = - TOX = 1 - -commands = python psutil/tests/__main__.py - -usedevelop = True - -[testenv:lint] -deps = flake8 -commands = flake8 -skip_install = True |
