From 38cb66b9c58db8eb5c567305086dca9b7b384fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 17 Apr 2020 18:32:54 +0100 Subject: Setup CI for tox 4 (#1551) --- src/tox/config/cli/env_var.py | 2 - src/tox/config/cli/ini.py | 4 +- src/tox/config/cli/parser.py | 10 +- src/tox/config/source/ini/__init__.py | 4 +- src/tox/config/source/ini/convert.py | 2 +- src/tox/execute/api.py | 8 +- src/tox/execute/local_sub_process.py | 175 --------------------- src/tox/execute/local_sub_process/__init__.py | 120 ++++++++++++++ .../execute/local_sub_process/read_via_thread.py | 54 +++++++ .../local_sub_process/read_via_thread_unix.py | 25 +++ .../local_sub_process/read_via_thread_windows.py | 39 +++++ src/tox/helper/build_requires.py | 2 - src/tox/helper/wheel_meta.py | 2 - src/tox/log/command.py | 3 - src/tox/log/env.py | 2 - src/tox/log/result.py | 2 +- src/tox/pytest.py | 33 ++-- src/tox/session/cmd/show_config.py | 5 +- src/tox/tox_env/api.py | 28 +++- src/tox/tox_env/python/api.py | 47 ++++-- src/tox/tox_env/python/virtual_env/api.py | 43 ++--- src/tox/util/__init__.py | 2 - src/tox/util/graph.py | 2 - src/tox/util/lock.py | 1 - src/tox/util/spinner.py | 2 - 25 files changed, 340 insertions(+), 277 deletions(-) delete mode 100644 src/tox/execute/local_sub_process.py create mode 100644 src/tox/execute/local_sub_process/__init__.py create mode 100644 src/tox/execute/local_sub_process/read_via_thread.py create mode 100644 src/tox/execute/local_sub_process/read_via_thread_unix.py create mode 100644 src/tox/execute/local_sub_process/read_via_thread_windows.py (limited to 'src') diff --git a/src/tox/config/cli/env_var.py b/src/tox/config/cli/env_var.py index 6cf5200a..ac06e64b 100644 --- a/src/tox/config/cli/env_var.py +++ b/src/tox/config/cli/env_var.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - import logging import os diff --git a/src/tox/config/cli/ini.py b/src/tox/config/cli/ini.py index b8408dba..c5ac7910 100644 --- a/src/tox/config/cli/ini.py +++ b/src/tox/config/cli/ini.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - import logging import os from pathlib import Path @@ -12,7 +10,7 @@ from tox.config.source.ini import Ini, IniLoader DEFAULT_CONFIG_FILE = Path(user_config_dir("tox")) / "config.ini" -class IniConfig(object): +class IniConfig: TOX_CONFIG_FILE_ENV_VAR = "TOX_CONFIG_FILE" STATE = {None: "failed to parse", True: "active", False: "missing"} diff --git a/src/tox/config/cli/parser.py b/src/tox/config/cli/parser.py index a3970bc0..3676386c 100644 --- a/src/tox/config/cli/parser.py +++ b/src/tox/config/cli/parser.py @@ -19,7 +19,7 @@ class ArgumentParserWithEnvAndConfig(ArgumentParser): def __init__(self, *args: Any, **kwargs: Any) -> None: self.file_config = IniConfig() kwargs["epilog"] = self.file_config.epilog - super(ArgumentParserWithEnvAndConfig, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def fix_defaults(self) -> None: for action in self._actions: @@ -55,7 +55,7 @@ class ArgumentParserWithEnvAndConfig(ArgumentParser): class HelpFormatter(ArgumentDefaultsHelpFormatter): def __init__(self, prog: str) -> None: - super(HelpFormatter, self).__init__(prog, max_help_position=42, width=240) + super().__init__(prog, max_help_position=42, width=240) def _get_help_string(self, action: Action) -> str: # noinspection PyProtectedMember @@ -120,14 +120,14 @@ class ToxParser(ArgumentParserWithEnvAndConfig): level_map = "|".join("{} - {}".format(c, logging.getLevelName(l)) for c, l in sorted(list(LEVELS.items()))) verbosity_group = self.add_argument_group( - "verbosity=verbose-quiet, default {}, map {}".format(logging.getLevelName(LEVELS[3]), level_map) + "verbosity=verbose-quiet, default {}, map {}".format(logging.getLevelName(LEVELS[3]), level_map), ) verbosity_exclusive = verbosity_group.add_mutually_exclusive_group() verbosity_exclusive.add_argument( - "-v", "--verbose", action="count", dest="verbose", help="increase verbosity", default=2 + "-v", "--verbose", action="count", dest="verbose", help="increase verbosity", default=2, ) verbosity_exclusive.add_argument( - "-q", "--quiet", action="count", dest="quiet", help="decrease verbosity", default=0 + "-q", "--quiet", action="count", dest="quiet", help="decrease verbosity", default=0, ) self.fix_defaults() diff --git a/src/tox/config/source/ini/__init__.py b/src/tox/config/source/ini/__init__.py index f342fbf7..cc298171 100644 --- a/src/tox/config/source/ini/__init__.py +++ b/src/tox/config/source/ini/__init__.py @@ -150,7 +150,7 @@ class IniLoader(Loader, StrConvert): def __repr__(self): return "{}(section={}, src={!r})".format( - type(self).__name__, self._section.name if self._section else self.name, self._src + type(self).__name__, self._section.name if self._section else self.name, self._src, ) def _load_raw(self, key, conf, as_name=None): @@ -193,4 +193,6 @@ class IniLoader(Loader, StrConvert): @property def section_name(self): + if self._section is None: + return None return self._section.name diff --git a/src/tox/config/source/ini/convert.py b/src/tox/config/source/ini/convert.py index 30394ce1..09db91c5 100644 --- a/src/tox/config/source/ini/convert.py +++ b/src/tox/config/source/ini/convert.py @@ -60,5 +60,5 @@ class StrConvert(Convert): return False else: raise TypeError( - "value {} cannot be transformed to bool, valid: {}".format(value, ", ".join(StrConvert.VALID_BOOL)) + "value {} cannot be transformed to bool, valid: {}".format(value, ", ".join(StrConvert.VALID_BOOL)), ) diff --git a/src/tox/execute/api.py b/src/tox/execute/api.py index a7e84de0..8a14b553 100644 --- a/src/tox/execute/api.py +++ b/src/tox/execute/api.py @@ -14,6 +14,7 @@ from .stream import CollectWrite ContentHandler = Callable[[bytes], None] Executor = Callable[[ExecuteRequest, ContentHandler, ContentHandler], int] +SIGINT = signal.CTRL_C_EVENT if sys.platform == "win32" else signal.SIGINT class ExecuteInstance: @@ -119,7 +120,8 @@ class Execute(ABC): is_main = threading.current_thread() == threading.main_thread() if is_main: # disable further interrupts until we finish this, main thread only - signal.signal(signal.SIGINT, signal.SIG_IGN) + if sys.platform != "win32": + signal.signal(SIGINT, signal.SIG_IGN) except KeyboardInterrupt: # pragma: no cover continue # pragma: no cover else: @@ -127,8 +129,8 @@ class Execute(ABC): exit_code = instance.interrupt() break finally: - if is_main: # restore signal handler on main thread - signal.signal(signal.SIGINT, signal.default_int_handler) + if is_main and sys.platform != "win32": # restore signal handler on main thread + signal.signal(SIGINT, signal.default_int_handler) finally: end = timer() result = Outcome(request, show_on_standard, exit_code, out.text, err.text, start, end, instance.cmd) diff --git a/src/tox/execute/local_sub_process.py b/src/tox/execute/local_sub_process.py deleted file mode 100644 index 4d38e524..00000000 --- a/src/tox/execute/local_sub_process.py +++ /dev/null @@ -1,175 +0,0 @@ -"""A execute that runs on local file system via subprocess-es""" -import logging -import os -import select -import shutil -import signal -import subprocess -import sys -from threading import Event, Thread -from typing import List, Optional, Sequence, Tuple, Type - -from .api import ContentHandler, Execute, ExecuteInstance, ExecuteRequest, Outcome - -WAIT_INTERRUPT = 0.3 -WAIT_TERMINATE = 0.2 -WAIT_GENERAL = 0.1 - - -class LocalSubProcessExecutor(Execute): - @staticmethod - def executor() -> Type[ExecuteInstance]: - return LocalSubProcessExecuteInstance - - -class LocalSubProcessExecuteInstance(ExecuteInstance): - def __init__(self, request: ExecuteRequest, out_handler: ContentHandler, err_handler: ContentHandler) -> None: - super().__init__(request, out_handler, err_handler) - self.process = None - self._cmd = [] # type: Optional[List[str]] - - @property - def cmd(self) -> Sequence[str]: - if not len(self._cmd): - executable = shutil.which(self.request.cmd[0], path=self.request.env["PATH"]) - if executable is None: - self._cmd = self.request.cmd # if failed to find leave as it is - else: - # else use expanded format - self._cmd = [executable, *self.request.cmd[1:]] - return self._cmd - - def run(self) -> int: - try: - self.process = process = subprocess.Popen( - self.cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=None if self.request.allow_stdin else subprocess.PIPE, - cwd=str(self.request.cwd), - env=self.request.env, - creationflags=( - subprocess.CREATE_NEW_PROCESS_GROUP - if sys.platform == "win32" - else 0 - # custom flag needed for Windows signal send ability (CTRL+C) - ), - ) - except OSError as exception: - exit_code = exception.errno - else: - with ReadViaThread(process.stderr, self.err_handler): - with ReadViaThread(process.stdout, self.out_handler): - # wait it out with interruptions to allow KeyboardInterrupt on Windows - while process.poll() is None: - try: - # note poll in general might deadlock if output large - # but we drain in background threads so not an issue here - process.wait(timeout=WAIT_GENERAL) - except subprocess.TimeoutExpired: - continue - exit_code = process.returncode - return exit_code - - def interrupt(self) -> int: - if self.process is not None: - out, err = self._handle_interrupt() # stop it and drain it - self._finalize_output(err, self.err_handler, out, self.out_handler) - return self.process.returncode - return Outcome.OK # pragma: no cover - - @staticmethod - def _finalize_output(err, err_handler, out, out_handler): - out_handler(out) - err_handler(err) - - def _handle_interrupt(self) -> Tuple[bytes, bytes]: - """A three level stop mechanism for children - INT -> TERM -> KILL""" - # communicate will wait for the app to stop, and then drain the standard streams and close them - proc = self.process - logging.error("got KeyboardInterrupt signal") - msg = "from {} {{}} pid {}".format(os.getpid(), proc.pid) - if proc.poll() is None: # still alive, first INT - logging.warning("KeyboardInterrupt %s", msg.format("SIGINT")) - proc.send_signal(signal.CTRL_C_EVENT if sys.platform == "win32" else signal.SIGINT) - try: - out, err = proc.communicate(timeout=WAIT_INTERRUPT) - except subprocess.TimeoutExpired: # if INT times out TERM - logging.warning("KeyboardInterrupt %s", msg.format("SIGTERM")) - proc.terminate() - try: - out, err = proc.communicate(timeout=WAIT_INTERRUPT) - except subprocess.TimeoutExpired: # if TERM times out KILL - logging.info("KeyboardInterrupt %s", msg.format("SIGKILL")) - proc.kill() - out, err = proc.communicate() - else: - out, err = proc.communicate() # just drain # pragma: no cover - return out, err - - -class ReadViaThread: - def __init__(self, stream, handler): - self.stream = stream - self.stop = Event() - self.thread = Thread(target=self._read_stream) - self.handler = handler - - def _read_stream(self): - file_no = self.stream.fileno() - while not (self.stream.closed or self.stop.is_set()): - # we need to drain the stream, but periodically give chance for the thread to break if the stop event has - # been set (this is so that an interrupt can be handled) - if self.stream_has_data(): - data = os.read(file_no, 1) - self.handler(data) - - def stream_has_data(self): - # TODO: select is UNIX only supported, for WINDOWS - # @zooba - # You need to use overlapped IO, which is all exposed in CPython through some internal modules - # (_winapi and _overlapped). The basic idea (and I haven't written this using those modules before) is that - # you provide a structure that includes a thread event and Windows will trigger that event when it's done - # But what you want may be more easily done by waiting on the file handle. I *think* that will work - # normally for streams that don't have any data available - # multiprocessing has some code using _overlapped to set up an overlapped pipe - # asyncio also uses it for its subprocess support, I believe - # Obviously since these are internal modules there's no documentation on them ;) But they do reflect the - # Windows API calls pretty closely, so if you look up the functions on http://docs.microsoft.com - # then you'll get all the details. - read_available_list, _, __ = select.select([self.stream], [], [], 0.01) - return len(read_available_list) - - def __enter__(self): - self.thread.start() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - thrown = None - while True: - try: - self.stop.set() - while self.thread.is_alive(): - self.thread.join(WAIT_GENERAL) - except KeyboardInterrupt as exception: # pragma: no cover - thrown = exception # pragma: no cover - continue # pragma: no cover - else: - if thrown is not None: - raise thrown # pragma: no cover - else: # pragma: no cover - break # pragma: no cover - if exc_val is None: # drain what remains if we were not interrupted - try: - data = self.stream.read() - except ValueError: # pragma: no cover - pass # pragma: no cover - else: - while True: - try: - self.handler(data) - break - except KeyboardInterrupt as exception: # pragma: no cover - thrown = exception # pragma: no cover - if thrown is not None: - raise thrown # pragma: no cover diff --git a/src/tox/execute/local_sub_process/__init__.py b/src/tox/execute/local_sub_process/__init__.py new file mode 100644 index 00000000..c23718d4 --- /dev/null +++ b/src/tox/execute/local_sub_process/__init__.py @@ -0,0 +1,120 @@ +"""A execute that runs on local file system via subprocess-es""" +import logging +import os +import shutil +import sys +from subprocess import PIPE, TimeoutExpired +from typing import List, Optional, Sequence, Tuple, Type + +from ..api import SIGINT, ContentHandler, Execute, ExecuteInstance, ExecuteRequest, Outcome +from .read_via_thread import WAIT_GENERAL + +if sys.platform == "win32": + from asyncio.windows_utils import Popen # noqa # needs stdin/stdout handlers backed by overlapped IO + from .read_via_thread_windows import ReadViaThreadWindows as ReadViaThread + from subprocess import CREATE_NEW_PROCESS_GROUP + + CREATION_FLAGS = CREATE_NEW_PROCESS_GROUP # custom flag needed for Windows signal send ability (CTRL+C) + +else: + from subprocess import Popen + from .read_via_thread_unix import ReadViaThreadUnix as ReadViaThread + + CREATION_FLAGS = 0 + + +WAIT_INTERRUPT = 0.3 +WAIT_TERMINATE = 0.2 + + +class LocalSubProcessExecutor(Execute): + @staticmethod + def executor() -> Type[ExecuteInstance]: + return LocalSubProcessExecuteInstance + + +class LocalSubProcessExecuteInstance(ExecuteInstance): + def __init__(self, request: ExecuteRequest, out_handler: ContentHandler, err_handler: ContentHandler) -> None: + super().__init__(request, out_handler, err_handler) + self.process = None + self._cmd = [] # type: Optional[List[str]] + + @property + def cmd(self) -> Sequence[str]: + if not len(self._cmd): + executable = shutil.which(self.request.cmd[0], path=self.request.env["PATH"]) + if executable is None: + self._cmd = self.request.cmd # if failed to find leave as it is + else: + # else use expanded format + self._cmd = [executable, *self.request.cmd[1:]] + return self._cmd + + def run(self) -> int: + try: + self.process = process = Popen( + self.cmd, + stdout=PIPE, + stderr=PIPE, + stdin=None if self.request.allow_stdin else PIPE, + cwd=str(self.request.cwd), + env=self.request.env, + creationflags=CREATION_FLAGS, + ) + except OSError as exception: + exit_code = exception.errno + else: + with ReadViaThread(process.stderr, self.err_handler) as read_stderr: + with ReadViaThread(process.stdout, self.out_handler) as read_stdout: + if sys.platform == "win32": + process.stderr.read = read_stderr._drain_stream + process.stdout.read = read_stdout._drain_stream + # wait it out with interruptions to allow KeyboardInterrupt on Windows + while process.poll() is None: + try: + # note poll in general might deadlock if output large + # but we drain in background threads so not an issue here + process.wait(timeout=WAIT_GENERAL) + except TimeoutExpired: + continue + exit_code = process.returncode + return exit_code + + def interrupt(self) -> int: + if self.process is not None: + out, err = self._handle_interrupt() # stop it and drain it + self._finalize_output(err, self.err_handler, out, self.out_handler) + return self.process.returncode + return Outcome.OK # pragma: no cover + + @staticmethod + def _finalize_output(err, err_handler, out, out_handler): + out_handler(out) + err_handler(err) + + def _handle_interrupt(self) -> Tuple[bytes, bytes]: + """A three level stop mechanism for children - INT -> TERM -> KILL""" + # communicate will wait for the app to stop, and then drain the standard streams and close them + proc = self.process + logging.error("got KeyboardInterrupt signal") + msg = "from {} {{}} pid {}".format(os.getpid(), proc.pid) + if proc.poll() is None: # still alive, first INT + logging.warning("KeyboardInterrupt %s", msg.format("SIGINT")) + proc.send_signal(SIGINT) + try: + out, err = proc.communicate(timeout=WAIT_INTERRUPT) + except TimeoutExpired: # if INT times out TERM + logging.warning("KeyboardInterrupt %s", msg.format("SIGTERM")) + proc.terminate() + try: + out, err = proc.communicate(timeout=WAIT_INTERRUPT) + except TimeoutExpired: # if TERM times out KILL + logging.info("KeyboardInterrupt %s", msg.format("SIGKILL")) + proc.kill() + out, err = proc.communicate() + else: + try: + out, err = proc.communicate() # just drain # pragma: no cover + except IndexError: + out, err = b"", b"" + return out, err diff --git a/src/tox/execute/local_sub_process/read_via_thread.py b/src/tox/execute/local_sub_process/read_via_thread.py new file mode 100644 index 00000000..de1f0c2e --- /dev/null +++ b/src/tox/execute/local_sub_process/read_via_thread.py @@ -0,0 +1,54 @@ +from abc import ABC, abstractmethod +from threading import Event, Thread + +WAIT_GENERAL = 0.1 + + +class ReadViaThread(ABC): + def __init__(self, stream, handler): + self.stream = stream + self.stop = Event() + self.thread = Thread(target=self._read_stream) + self.handler = handler + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + thrown = None + while True: + try: + self.stop.set() + while self.thread.is_alive(): + self.thread.join(WAIT_GENERAL) + except KeyboardInterrupt as exception: # pragma: no cover + thrown = exception # pragma: no cover + continue # pragma: no cover + else: + if thrown is not None: + raise thrown # pragma: no cover + else: # pragma: no cover + break # pragma: no cover + if exc_val is None: # drain what remains if we were not interrupted + try: + data = self._drain_stream() + except ValueError: # pragma: no cover + pass # pragma: no cover + else: + while True: + try: + self.handler(data) + break + except KeyboardInterrupt as exception: # pragma: no cover + thrown = exception # pragma: no cover + if thrown is not None: + raise thrown # pragma: no cover + + @abstractmethod + def _read_stream(self): + raise NotImplementedError + + @abstractmethod + def _drain_stream(self): + raise NotImplementedError diff --git a/src/tox/execute/local_sub_process/read_via_thread_unix.py b/src/tox/execute/local_sub_process/read_via_thread_unix.py new file mode 100644 index 00000000..b7882449 --- /dev/null +++ b/src/tox/execute/local_sub_process/read_via_thread_unix.py @@ -0,0 +1,25 @@ +import os +import select + +from .read_via_thread import ReadViaThread + + +class ReadViaThreadUnix(ReadViaThread): + def __init__(self, stream, handler): + super().__init__(stream, handler) + self.file_no = self.stream.fileno() + + def _read_stream(self): + while not (self.stream.closed or self.stop.is_set()): + # we need to drain the stream, but periodically give chance for the thread to break if the stop event has + # been set (this is so that an interrupt can be handled) + if self.has_bytes(): + data = os.read(self.file_no, 1) + self.handler(data) + + def has_bytes(self): + read_available_list, _, __ = select.select([self.stream], [], [], 0.01) + return len(read_available_list) + + def _drain_stream(self): + return self.stream.read() diff --git a/src/tox/execute/local_sub_process/read_via_thread_windows.py b/src/tox/execute/local_sub_process/read_via_thread_windows.py new file mode 100644 index 00000000..0bdce5a3 --- /dev/null +++ b/src/tox/execute/local_sub_process/read_via_thread_windows.py @@ -0,0 +1,39 @@ +from asyncio.windows_utils import BUFSIZE, PipeHandle + +import _overlapped + +from .read_via_thread import ReadViaThread + + +class ReadViaThreadWindows(ReadViaThread): + def __init__(self, stream, handler): + super().__init__(stream, handler) + self.closed = False + assert isinstance(stream, PipeHandle) + + def _read_stream(self): + ov = None + while not self.stop.is_set(): + if ov is None: + ov = _overlapped.Overlapped(0) + try: + ov.ReadFile(self.stream.handle, 1) + except BrokenPipeError: + self.closed = True + return + data = ov.getresult(10) # wait for 10ms + ov = None + self.handler(data) + + def _drain_stream(self): + length, result = 0 if self.closed else 1, b"" + while 0 < length <= BUFSIZE: + ov = _overlapped.Overlapped(0) + buffer = bytes(BUFSIZE) + try: + ov.ReadFileInto(self.stream.handle, buffer) + length = ov.getresult() + result += buffer[:length] + except BrokenPipeError: + break + return result diff --git a/src/tox/helper/build_requires.py b/src/tox/helper/build_requires.py index 01ae010c..6f47c90b 100644 --- a/src/tox/helper/build_requires.py +++ b/src/tox/helper/build_requires.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import json import sys diff --git a/src/tox/helper/wheel_meta.py b/src/tox/helper/wheel_meta.py index 1359513c..9f512253 100644 --- a/src/tox/helper/wheel_meta.py +++ b/src/tox/helper/wheel_meta.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import json import sys diff --git a/src/tox/log/command.py b/src/tox/log/command.py index 00e6ee96..6a3d34bf 100644 --- a/src/tox/log/command.py +++ b/src/tox/log/command.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import, unicode_literals - - class CommandLog: """Report commands interacting with third party tools""" diff --git a/src/tox/log/env.py b/src/tox/log/env.py index e5921b95..e358bf00 100644 --- a/src/tox/log/env.py +++ b/src/tox/log/env.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - from copy import copy from .command import CommandLog diff --git a/src/tox/log/result.py b/src/tox/log/result.py index b65a1120..1bfca3a5 100644 --- a/src/tox/log/result.py +++ b/src/tox/log/result.py @@ -12,7 +12,7 @@ from .env import EnvLog class ResultLog: """The result of a tox session""" - def __init__(self,): + def __init__(self): command_log = [] self.command_log = CommandLog(command_log) self.content = { diff --git a/src/tox/pytest.py b/src/tox/pytest.py index 2495c440..5a13e505 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -35,6 +35,7 @@ def check_os_environ(): new = os.environ extra = {k: new[k] for k in set(new) - set(old)} + extra.pop("PLAT", None) miss = {k: old[k] for k in set(old) - set(new)} diff = { "{} = {} vs {}".format(k, old[k], new[k]) @@ -76,22 +77,22 @@ class ToxProject: self.path = path # type: Path self._capsys = capsys self.monkeypatch = monkeypatch - - def _handle_level(of_path: Path, content: Dict[str, Any]) -> None: - for key, value in content.items(): - if not isinstance(key, str): - raise TypeError("{!r} at {}".format(key, of_path)) # pragma: no cover - at_path = of_path / key - if isinstance(value, dict): - at_path.mkdir(exist_ok=True) - _handle_level(at_path, value) - elif isinstance(value, str): - at_path.write_text(textwrap.dedent(value)) - else: - msg = "could not handle {} with content {!r}".format(at_path / key, value) # pragma: no cover - raise TypeError(msg) # pragma: no cover - - _handle_level(self.path, files) + self._setup_files(self.path, files) + + @staticmethod + def _setup_files(dest: Path, content: Dict[str, Any]) -> None: + for key, value in content.items(): + if not isinstance(key, str): + raise TypeError("{!r} at {}".format(key, dest)) # pragma: no cover + at_path = dest / key + if isinstance(value, dict): + at_path.mkdir(exist_ok=True) + ToxProject._setup_files(at_path, value) + elif isinstance(value, str): + at_path.write_text(textwrap.dedent(value)) + else: + msg = "could not handle {} with content {!r}".format(at_path / key, value) # pragma: no cover + raise TypeError(msg) # pragma: no cover @property def structure(self): diff --git a/src/tox/session/cmd/show_config.py b/src/tox/session/cmd/show_config.py index b90baac4..98a36690 100644 --- a/src/tox/session/cmd/show_config.py +++ b/src/tox/session/cmd/show_config.py @@ -1,4 +1,3 @@ -import os from typing import Any, List, Union from tox.config.cli.parser import ToxParser @@ -38,8 +37,8 @@ def print_conf(conf: ConfigSet) -> None: value = conf[key] result = str_conf_value(value) if isinstance(result, list): - result = "{}{}".format(os.linesep, os.linesep.join(" {}".format(i) for i in result)) - print("{} = {}".format(key, result)) + result = "{}{}".format("\n", "\n".join(" {}".format(i) for i in result)) + print("{} ={}{}".format(key, " " if result != "" and not result.startswith("\n") else "", result)) unused = conf.unused() if unused: print("!!! unused: {}".format(",".join(unused))) diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py index b8bb2079..ba9be1fc 100644 --- a/src/tox/tox_env/api.py +++ b/src/tox/tox_env/api.py @@ -2,9 +2,10 @@ import itertools import logging import os import shutil +import sys from abc import ABC, abstractmethod from pathlib import Path -from typing import Dict, List, Optional, Sequence, Set, Union, cast +from typing import Dict, List, Optional, Sequence, Union, cast from tox.config.sets import ConfigSet from tox.execute.api import Execute @@ -12,6 +13,23 @@ from tox.execute.request import ExecuteRequest from .cache import Cache +if sys.platform == "win32": + PASS_ENV_ALWAYS = [ + "SYSTEMDRIVE", # needed for pip6 + "SYSTEMROOT", # needed for python's crypto module + "PATHEXT", # needed for discovering executables + "COMSPEC", # needed for distutils cygwin compiler + "PROCESSOR_ARCHITECTURE", # platform.machine() + "USERPROFILE", # needed for `os.path.expanduser()` + "MSYSTEM", # controls paths printed format + "TEMP", + "TMP", + ] +else: + PASS_ENV_ALWAYS = [ + "TMPDIR", + ] + class ToxEnv(ABC): def __init__(self, conf: ConfigSet, core: ConfigSet, options, executor: Execute): @@ -82,14 +100,16 @@ class ToxEnv(ABC): @property def environment_variables(self) -> Dict[str, str]: result = {} # type:Dict[str, str] - pass_env = self.conf["pass_env"] # type:Set[str] - set_env = self.conf["set_env"] # type:Dict[str, str] + pass_env = self.conf["pass_env"] # type: List[str] + pass_env.extend(PASS_ENV_ALWAYS) + + set_env = self.conf["set_env"] # type: Dict[str, str] for key, value in os.environ.items(): if key in pass_env: result[key] = value result.update(set_env) result["PATH"] = os.pathsep.join( - itertools.chain((str(i) for i in self._paths), os.environ.get("PATH", "").split(os.pathsep)) + itertools.chain((str(i) for i in self._paths), os.environ.get("PATH", "").split(os.pathsep)), ) return result diff --git a/src/tox/tox_env/python/api.py b/src/tox/tox_env/python/api.py index 6136e7ec..5ce9e553 100644 --- a/src/tox/tox_env/python/api.py +++ b/src/tox/tox_env/python/api.py @@ -1,11 +1,13 @@ import sys from abc import ABC, abstractmethod +from argparse import Namespace from pathlib import Path -from typing import Any, List, Union +from typing import Any, List, Union, cast from packaging.requirements import Requirement from virtualenv.discovery.builtin import get_interpreter from virtualenv.discovery.py_spec import PythonSpec +from virtualenv.run import AppDataAction, CreatorSelector from tox.config.sets import ConfigSet from tox.execute.api import Execute @@ -19,6 +21,12 @@ class Python(ToxEnv, ABC): self._python = None self._python_search_done = False + @property + def py_info(self): + if self._python is None: + self._find_base_python() + return self._python + def register_config(self): super().register_config() self.conf.add_config( @@ -45,21 +53,16 @@ class Python(ToxEnv, ABC): If we have the python we just need to look at the last path under prefix. Debian derivatives change the site-packages to dist-packages, so we need to fix it for site-packages. """ - python = self._find_base_python() - site_at = next(Path(p) for p in reversed(python.path) if p.startswith(python.prefix)).relative_to( - Path(python.prefix) - ) - return self.conf["env_dir"] / site_at.parent / "site-packages" + return self.py_info.purelib def setup(self) -> None: """setup a virtual python environment""" super().setup() - python = self._find_base_python() - conf = self.python_cache(python) + conf = self.python_cache() with self._cache.compare(conf, Python.__name__) as (eq, old): if eq is False: - self.create_python_env(python) - self._paths = self.paths(python) + self.create_python_env() + self._paths = self.paths() def _find_base_python(self): base_pythons = self.conf["base_python"] @@ -68,7 +71,23 @@ class Python(ToxEnv, ABC): for base_python in base_pythons: python = self.get_python(base_python) if python is not None: - self._python = python + env_dir = cast(Path, self.conf["env_dir"]) + selector = CreatorSelector.for_interpreter(python) + info = selector.describe( + Namespace( + dest=env_dir, + clear=False, + system_site=False, + app_data=AppDataAction.default(), + meta=selector.key_to_meta[ + next( + name for name, value in selector.key_to_class.items() if value == selector.describe + ) + ], + ), + python, + ) + self._python = info break if self._python is None: raise NoInterpreter(base_pythons) @@ -95,15 +114,15 @@ class Python(ToxEnv, ABC): return False @abstractmethod - def python_cache(self, python) -> Any: + def python_cache(self) -> Any: raise NotImplementedError @abstractmethod - def create_python_env(self, python) -> List[Path]: + def create_python_env(self) -> List[Path]: raise NotImplementedError @abstractmethod - def paths(self, python) -> List[Path]: + def paths(self) -> List[Path]: raise NotImplementedError @abstractmethod diff --git a/src/tox/tox_env/python/virtual_env/api.py b/src/tox/tox_env/python/virtual_env/api.py index a3a30001..2c1ae0c9 100644 --- a/src/tox/tox_env/python/virtual_env/api.py +++ b/src/tox/tox_env/python/virtual_env/api.py @@ -1,11 +1,9 @@ -import shutil import sys from abc import ABC from pathlib import Path from typing import List, Sequence, Union, cast from packaging.requirements import Requirement -from virtualenv.discovery.py_info import PythonInfo from tox.config.sets import ConfigSet from tox.execute.api import Outcome @@ -14,57 +12,36 @@ from tox.execute.local_sub_process import LocalSubProcessExecutor from ..api import Python -def copy_overwrite(src: Path, dest: Path): - if dest.exists(): - shutil.rmtree(str(dest)) - if src.is_dir(): - if not dest.is_dir(): - dest.mkdir(parents=True) - for file_name in src.iterdir(): - copy_overwrite(file_name, dest / file_name.name) - else: - shutil.copyfile(str(src), str(dest)) - - class VirtualEnv(Python, ABC): def __init__(self, conf: ConfigSet, core: ConfigSet, options): super().__init__(conf, core, options, LocalSubProcessExecutor()) - def create_python_env(self, python: PythonInfo): - core_cmd = self.core_cmd(python) + def create_python_env(self): + core_cmd = self.core_cmd() env_dir = cast(Path, self.conf["env_dir"]) cmd = core_cmd + ("--clear", env_dir) result = self.execute(cmd=cmd, allow_stdin=False) result.assert_success(self.logger) - @staticmethod - def core_cmd(python): + def core_cmd(self): core_cmd = ( sys.executable, "-m", "virtualenv", "--no-download", "--python", - python.executable, + self.py_info.interpreter.system_executable, ) return core_cmd - @staticmethod - def get_bin(folder: Path) -> Path: - return next(p for p in folder.iterdir() if p.name in ("bin", "Script")) - - @staticmethod - def get_site_packages(folder: Path) -> Path: - lib = next(next(i for i in folder.iterdir() if i.name in ("lib", "Lib")).iterdir()) - return lib / "site-packages" - - def paths(self, python: PythonInfo) -> List[Path]: + def paths(self) -> List[Path]: + """Paths to add to the executable""" # we use the original executable as shims may be somewhere else - host_postfix = Path(python.original_executable).relative_to(python.prefix).parent - return [self.conf["env_dir"] / host_postfix] + return list({self.py_info.bin_dir, self.py_info.script_dir}) - def python_cache(self, python: PythonInfo): - return {"version_info": list(python.version_info), "executable": python.executable} + def python_cache(self): + base_python = self.py_info.interpreter + return {"version_info": list(base_python.version_info), "executable": base_python.executable} def install_python_packages( self, packages: List[Union[Requirement, Path]], no_deps: bool = False, develop=False, force_reinstall=False, diff --git a/src/tox/util/__init__.py b/src/tox/util/__init__.py index c72dea0f..731b7ecb 100644 --- a/src/tox/util/__init__.py +++ b/src/tox/util/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - import os from contextlib import contextmanager diff --git a/src/tox/util/graph.py b/src/tox/util/graph.py index db258409..4c1b1089 100644 --- a/src/tox/util/graph.py +++ b/src/tox/util/graph.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - from collections import OrderedDict, defaultdict diff --git a/src/tox/util/lock.py b/src/tox/util/lock.py index 1234bde4..f5f279ae 100644 --- a/src/tox/util/lock.py +++ b/src/tox/util/lock.py @@ -1,5 +1,4 @@ """holds locking functionality that works across processes""" -from __future__ import absolute_import, unicode_literals import logging from contextlib import contextmanager diff --git a/src/tox/util/spinner.py b/src/tox/util/spinner.py index 7f4240e3..5e926b3a 100644 --- a/src/tox/util/spinner.py +++ b/src/tox/util/spinner.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- """A minimal non-colored version of https://pypi.org/project/halo, to track list progress""" -from __future__ import absolute_import, unicode_literals import os import sys -- cgit v1.2.1