From 64b4318bce1a2ba8dcc0b8f7cbaefeb029c12419 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Mon, 27 Mar 2023 23:37:16 +0200 Subject: Win: fix running tests in a virtual environment (#2216) On windows, starting with python 3.7, virtual environments use a venvlauncher startup process This does not play well when counting spawned processes or when relying on the pid of the spawned process to do some checks e.g. connection check per pid This commit detects this situation and uses the base python executable to spawn processes when required. Signed-off-by: mayeut --- .github/workflows/build.yml | 60 +++++++++--------------------------------- psutil/tests/__init__.py | 35 ++++++++++++++---------- psutil/tests/test_misc.py | 2 ++ psutil/tests/test_process.py | 7 ++--- psutil/tests/test_testutils.py | 4 ++- pyproject.toml | 6 ----- 6 files changed, 42 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e2d015c1..eb6996da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,14 +22,21 @@ name: build jobs: # Linux + macOS + Windows Python 3 py3: - name: py3-${{ matrix.os }} + name: py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'windows') && matrix.archs || 'all' }} runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-2019] - # python: ["3.6", "3.11"] + include: + - os: ubuntu-latest + archs: "x86_64 i686" + - os: macos-12 + archs: "x86_64 arm64" + - os: windows-2019 + archs: "AMD64" + - os: windows-2019 + archs: "x86" steps: - name: Cancel previous runs @@ -44,6 +51,8 @@ jobs: - name: Create wheels + run tests uses: pypa/cibuildwheel@v2.11.2 + env: + CIBW_ARCHS: "${{ matrix.archs }}" - name: Upload wheels uses: actions/upload-artifact@v3 @@ -58,51 +67,6 @@ jobs: python setup.py sdist mv dist/psutil*.tar.gz wheelhouse/ - # Windows cp37+ tests - # psutil tests do not like running from a virtualenv with python>=3.7 so - # not using cibuildwheel for those. run them "manually" with this job. - py3-windows-tests: - name: py3-windows-test-${{ matrix.python }}-${{ matrix.architecture }} - needs: py3 - runs-on: windows-2019 - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - architecture: ["x86", "x64"] - - steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python }}" - architecture: "${{ matrix.architecture }}" - - - name: Download wheels - uses: actions/download-artifact@v3 - with: - name: wheels - path: wheelhouse - - - name: Run tests - run: | - mkdir .tests - cd .tests - pip install $(find ../wheelhouse -name '*-cp36-abi3-${{ matrix.architecture == 'x86' && 'win32' || 'win_amd64'}}.whl')[test] - export PYTHONWARNINGS=always - export PYTHONUNBUFFERED=1 - export PSUTIL_DEBUG=1 - python ../psutil/tests/runner.py - python ../psutil/tests/test_memleaks.py - shell: bash - # Linux + macOS + Python 2 py2: name: py2-${{ matrix.os }} diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 61a06e21..e3766eea 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -41,7 +41,6 @@ from socket import SOCK_STREAM import psutil from psutil import AIX -from psutil import FREEBSD from psutil import LINUX from psutil import MACOS from psutil import POSIX @@ -80,8 +79,8 @@ if POSIX: __all__ = [ # constants 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', - 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX', - 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', + 'PYPY', 'PYTHON_EXE', 'PYTHON_EXE_ENV', 'ROOT_DIR', 'SCRIPTS_DIR', + 'TESTFN_PREFIX', 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT', "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", @@ -240,13 +239,21 @@ def _get_py_exe(): else: return exe - if GITHUB_ACTIONS: - if PYPY: - return which("pypy3") if PY3 else which("pypy") - elif FREEBSD: - return os.path.realpath(sys.executable) - else: - return which('python') + env = os.environ.copy() + + # On Windows, starting with python 3.7, virtual environments use a + # venv launcher startup process. This does not play well when + # counting spawned processes, or when relying on the PID of the + # spawned process to do some checks, e.g. connections check per PID. + # Let's use the base python in this case. + base = getattr(sys, "_base_executable", None) + if WINDOWS and sys.version_info >= (3, 7) and base is not None: + # We need to set __PYVENV_LAUNCHER__ to sys.executable for the + # base python executable to know about the environment. + env["__PYVENV_LAUNCHER__"] = sys.executable + return base, env + elif GITHUB_ACTIONS: + return sys.executable, env elif MACOS: exe = \ attempt(sys.executable) or \ @@ -255,14 +262,14 @@ def _get_py_exe(): attempt(psutil.Process().exe()) if not exe: raise ValueError("can't find python exe real abspath") - return exe + return exe, env else: exe = os.path.realpath(sys.executable) assert os.path.exists(exe), exe - return exe + return exe, env -PYTHON_EXE = _get_py_exe() +PYTHON_EXE, PYTHON_EXE_ENV = _get_py_exe() DEVNULL = open(os.devnull, 'r+') atexit.register(DEVNULL.close) @@ -351,7 +358,7 @@ def spawn_testproc(cmd=None, **kwds): kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) kwds.setdefault("cwd", os.getcwd()) - kwds.setdefault("env", os.environ) + kwds.setdefault("env", PYTHON_EXE_ENV) if WINDOWS: # Prevents the subprocess to open error dialogs. This will also # cause stderr to be suppressed, which is suboptimal in order diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 44922216..afa60b1c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -45,6 +45,7 @@ from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV from psutil.tests import SCRIPTS_DIR from psutil.tests import PsutilTestCase from psutil.tests import mock @@ -820,6 +821,7 @@ class TestScripts(PsutilTestCase): @staticmethod def assert_stdout(exe, *args, **kwargs): + kwargs.setdefault("env", PYTHON_EXE_ENV) exe = '%s' % os.path.join(SCRIPTS_DIR, exe) cmd = [PYTHON_EXE, exe] for arg in args: diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 26869e98..df6a84c3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -52,6 +52,7 @@ from psutil.tests import HAS_THREADS from psutil.tests import MACOS_11PLUS from psutil.tests import PYPY from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV from psutil.tests import PsutilTestCase from psutil.tests import ThreadTask from psutil.tests import call_until @@ -1543,7 +1544,7 @@ class TestPopen(PsutilTestCase): # Not sure what to do though. cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] with psutil.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: + stderr=subprocess.PIPE, env=PYTHON_EXE_ENV) as proc: proc.name() proc.cpu_times() proc.stdin @@ -1559,7 +1560,7 @@ class TestPopen(PsutilTestCase): with psutil.Popen([PYTHON_EXE, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE) as proc: + stdin=subprocess.PIPE, env=PYTHON_EXE_ENV) as proc: proc.communicate() assert proc.stdout.closed assert proc.stderr.closed @@ -1572,7 +1573,7 @@ class TestPopen(PsutilTestCase): # diverges from that. cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] with psutil.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: + stderr=subprocess.PIPE, env=PYTHON_EXE_ENV) as proc: proc.terminate() proc.wait() self.assertRaises(psutil.NoSuchProcess, proc.terminate) diff --git a/psutil/tests/test_testutils.py b/psutil/tests/test_testutils.py index 0298ea4e..e757e017 100755 --- a/psutil/tests/test_testutils.py +++ b/psutil/tests/test_testutils.py @@ -30,6 +30,7 @@ from psutil.tests import CI_TESTING from psutil.tests import COVERAGE from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV from psutil.tests import PsutilTestCase from psutil.tests import TestMemoryLeak from psutil.tests import bind_socket @@ -260,7 +261,8 @@ class TestProcessUtils(PsutilTestCase): terminate(p) # by psutil.Popen cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] - p = psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=PYTHON_EXE_ENV) terminate(p) self.assertProcessGone(p) terminate(p) diff --git a/pyproject.toml b/pyproject.toml index 435cc989..c8cb62af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,3 @@ test-command = [ [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] - -[tool.cibuildwheel.windows] -# psutil tests do not like running from a virtualenv with python>=3.7 -# restrict build & tests to cp36 -# cp36-abi3 wheels will need to be tested outside cibuildwheel for python>=3.7 -build = "cp36-*" -- cgit v1.2.1