Add keyboard interrupt support for parallel

Part of this unify the sequential and parallel execution engines.

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

6 files changed, 71 insertions, 104 deletions

diff --git a/src/tox/util/pep517/backend.py b/src/tox/util/pep517/backend.py
index b1abdfbb..a20d8b7a 100644
--- a/src/tox/util/pep517/backend.py
+++ b/src/tox/util/pep517/backend.py
@@ -4,6 +4,10 @@ import sys
 import traceback
 
 
+class MissingCommand(TypeError):
+    """Missing command"""
+
+
 class BackendProxy:
     def __init__(self, backend_module, backend_obj):
         self.backend_module = backend_module
@@ -16,27 +20,12 @@ class BackendProxy:
     def __call__(self, name, *args, **kwargs):
         on_object = self if name.startswith("_") else self.backend
         if not hasattr(on_object, name):
-            raise TypeError(f"{on_object!r} has no attribute {name!r}")
+            raise MissingCommand(f"{on_object!r} has no attribute {name!r}")
         return getattr(on_object, name)(*args, **kwargs)
 
     def _exit(self):  # noqa
         return 0
 
-    def _commands(self):
-        result = ["_commands", "_exit"]
-        result.extend(
-            k
-            for k in {
-                "build_sdist",
-                "build_wheel",
-                "prepare_metadata_for_build_wheel",
-                "get_requires_for_build_wheel",
-                "get_requires_for_build_sdist",
-            }
-            if hasattr(self.backend, k)
-        )
-        return result
-
 
 def flush():
     sys.stderr.flush()
@@ -60,7 +49,6 @@ def run(argv):
         except Exception:  # noqa
             # ignore messages that are not valid JSON and contain a valid result path
             print(f"Backend: incorrect request to backend: {message}", file=sys.stderr)
-            traceback.print_exc()
             flush()
         else:
             result = {}
@@ -71,11 +59,14 @@ def run(argv):
                 result["return"] = outcome
                 if cmd == "_exit":
                     break
-            except Exception as exception:
-                traceback.print_exc()
+            except BaseException as exception:
                 result["code"] = exception.code if isinstance(exception, SystemExit) else 1
                 result["exc_type"] = exception.__class__.__name__
                 result["exc_msg"] = str(exception)
+                if not isinstance(exception, MissingCommand):  # for missing command do not print stack
+                    traceback.print_exc()
+                if not isinstance(exception, Exception):  # allow SystemExit/KeyboardInterrupt to go through
+                    raise
             finally:
                 try:
                     with open(result_file, "wt") as file_handler:
@@ -83,8 +74,8 @@ def run(argv):
                 except Exception:  # noqa
                     traceback.print_exc()
                 finally:
-                    print(f"Backend: Wrote response {result} to {result_file}")
-                    flush()
+                    print(f"Backend: Wrote response {result} to {result_file}")  # used as done marker by frontend
+                    flush()  # pragma: no branch
         if reuse_process is False:  # pragma: no branch # no test for reuse process in root test env
             break
     return 0
diff --git a/src/tox/util/pep517/backend.pyi b/src/tox/util/pep517/backend.pyi
index fe49bb17..4eb09d0c 100644
--- a/src/tox/util/pep517/backend.pyi
+++ b/src/tox/util/pep517/backend.pyi
@@ -1,5 +1,7 @@
 """Handles communication on the backend side between frontend and backend"""
-from typing import Any, List, Optional, Sequence
+from typing import Any, Optional, Sequence
+
+class MissingCommand(TypeError): ...
 
 class BackendProxy:
     backend_module: str
@@ -8,7 +10,6 @@ class BackendProxy:
     def __init__(self, backend_module: str, backend_obj: Optional[str]) -> None: ...
     def __call__(self, name: str, *args: Any, **kwargs: Any) -> Any: ...
     def _exit(self) -> None: ...
-    def _commands(self) -> List[str]: ...
 
 def run(argv: Sequence[str]) -> int: ...
 def flush() -> None: ...
diff --git a/src/tox/util/pep517/frontend.py b/src/tox/util/pep517/frontend.py
index 0970d571..e5a500b7 100644
--- a/src/tox/util/pep517/frontend.py
+++ b/src/tox/util/pep517/frontend.py
@@ -6,7 +6,7 @@ from contextlib import contextmanager
 from pathlib import Path
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from time import sleep
-from typing import Any, Dict, Iterator, List, NamedTuple, NoReturn, Optional, Set, Tuple, cast
+from typing import Any, Dict, Iterator, List, NamedTuple, NoReturn, Optional, Tuple, cast
 from zipfile import ZipFile
 
 import toml
@@ -62,9 +62,9 @@ class BackendFailed(RuntimeError):
         super().__init__()
         self.out = out
         self.err = err
-        self.code: int = result["code"]
-        self.exc_type: str = result["exc_type"]
-        self.exc_msg: str = result["exc_msg"]
+        self.code: int = result.get("code", -2)
+        self.exc_type: str = result.get("exc_type", "missing Exception type")
+        self.exc_msg: str = result.get("exc_msg", "missing Exception message")
 
     def __str__(self) -> str:
 
@@ -89,12 +89,6 @@ class BackendFailed(RuntimeError):
 class Frontend(ABC):
     LEGACY_BUILD_BACKEND: str = "setuptools.build_meta:__legacy__"
     LEGACY_REQUIRES: Tuple[Requirement, ...] = (Requirement("setuptools >= 40.8.0"), Requirement("wheel"))
-    ALWAYS_EXIST = (
-        "_commands",
-        "_exit",
-        "build_wheel",
-        "build_sdist",
-    )
 
     def __init__(
         self,
@@ -110,7 +104,6 @@ class Frontend(ABC):
         self._backend_module = backend_module
         self._backend_obj = backend_obj
         self._requires = requires
-        self._commands: Optional[Set[str]] = None
         self._reuse_backend = reuse_backend
 
     @classmethod
@@ -145,7 +138,6 @@ class Frontend(ABC):
             cmd="build_sdist",
             sdist_directory=sdist_directory,
             config_settings=config_settings,
-            missing=None,  #
         )
         if not isinstance(basename, str):
             self._unexpected_response("build_sdist", basename, str, out, err)
@@ -163,7 +155,6 @@ class Frontend(ABC):
             wheel_directory=wheel_directory,
             config_settings=config_settings,
             metadata_directory=metadata_directory,
-            missing=None,
         )
         if not isinstance(basename, str):
             self._unexpected_response("build_wheel", basename, str, out, err)
@@ -185,16 +176,17 @@ class Frontend(ABC):
         if metadata_directory.exists():  # start with fresh
             shutil.rmtree(metadata_directory)
         metadata_directory.mkdir(parents=True, exist_ok=True)
-        basename, out, err = self._send(
-            cmd="prepare_metadata_for_build_wheel",
-            metadata_directory=metadata_directory,
-            config_settings=config_settings,
-            missing=object,
-        )
-        if basename is not object and not isinstance(basename, str):
-            self._unexpected_response("prepare_metadata_for_build_wheel", basename, str, out, err)
-        if basename is object:  # if backend does not provide it acquire it from the wheel
+        try:
+            basename, out, err = self._send(
+                cmd="prepare_metadata_for_build_wheel",
+                metadata_directory=metadata_directory,
+                config_settings=config_settings,
+            )
+        except BackendFailed:
+            # if backend does not provide it acquire it from the wheel
             basename, err, out = self._metadata_from_built_wheel(config_settings, metadata_directory)
+        if not isinstance(basename, str):
+            self._unexpected_response("prepare_metadata_for_build_wheel", basename, str, out, err)
         result = metadata_directory / basename
         return MetadataForBuildWheelResult(result, out, err)
 
@@ -231,7 +223,10 @@ class Frontend(ABC):
     def get_requires_for_build_wheel(
         self, config_settings: Optional[ConfigSettings] = None
     ) -> RequiresBuildWheelResult:
-        result, out, err = self._send(cmd="get_requires_for_build_wheel", config_settings=config_settings, missing=[])
+        try:
+            result, out, err = self._send(cmd="get_requires_for_build_wheel", config_settings=config_settings)
+        except BackendFailed as exc:
+            result, out, err = [], exc.out, exc.err
         if not isinstance(result, list) or not all(isinstance(i, str) for i in result):
             self._unexpected_response("get_requires_for_build_wheel", result, "list of string", out, err)
         return RequiresBuildWheelResult(tuple(Requirement(r) for r in cast(List[str], result)), out, err)
@@ -239,21 +234,15 @@ class Frontend(ABC):
     def get_requires_for_build_sdist(
         self, config_settings: Optional[ConfigSettings] = None
     ) -> RequiresBuildSdistResult:
-        result, out, err = self._send(cmd="get_requires_for_build_sdist", config_settings=config_settings, missing=[])
+        try:
+            result, out, err = self._send(cmd="get_requires_for_build_sdist", config_settings=config_settings)
+        except BackendFailed as exc:
+            result, out, err = [], exc.out, exc.err
         if not isinstance(result, list) or not all(isinstance(i, str) for i in result):
             self._unexpected_response("get_requires_for_build_sdist", result, "list of string", out, err)
         return RequiresBuildSdistResult(tuple(Requirement(r) for r in cast(List[str], result)), out, err)
 
-    @property
-    def commands(self) -> Set[str]:
-        if self._commands is None:
-            raw, _, __ = self._send("_commands", missing=[])
-            self._commands = set(raw)
-        return self._commands
-
-    def _send(self, cmd: str, missing: Any, **kwargs: Any) -> Tuple[Any, str, str]:
-        if cmd not in self.ALWAYS_EXIST and cmd not in self.commands:
-            return missing, "", ""
+    def _send(self, cmd: str, **kwargs: Any) -> Tuple[Any, str, str]:
         with NamedTemporaryFile(prefix=f"pep517_{cmd}-") as result_file_marker:
             result_file = Path(result_file_marker.name).with_suffix(".json")
             msg = json.dumps(
diff --git a/src/tox/util/signal.py b/src/tox/util/signal.py
deleted file mode 100644
index 6d413a3b..00000000
--- a/src/tox/util/signal.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import logging
-import threading
-from signal import SIGINT, Handlers, Signals, signal
-from types import FrameType, TracebackType
-from typing import Callable, Optional, Type, Union
-
-
-class DelayedSignal:
-    def __init__(self, of: Signals = SIGINT) -> None:
-        self._of = of
-        self._signal: Optional[Signals] = None
-        self._frame: Optional[FrameType] = None
-        self._old_handler: Union[Callable[[Signals, FrameType], None], int, Handlers, None] = None
-
-    def __enter__(self) -> "DelayedSignal":
-        self._signal, self._frame = None, None
-        if threading.current_thread() == threading.main_thread():  # signals are always handled on the main thread only
-            self._old_handler = signal(self._of, self._handler)
-        return self
-
-    def __exit__(
-        self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]
-    ) -> None:
-        try:
-            if self._signal is not None and self._frame is not None and callable(self._old_handler):
-                logging.debug("Handling delayed %s", self._signal)
-                self._old_handler(self._signal, self._frame)
-        finally:
-            if self._old_handler is not None:
-                signal(self._of, self._old_handler)
-
-    def _handler(self, sig: Signals, frame: FrameType) -> None:
-        logging.debug("Received %s, delaying it", sig)
-        self._signal, self._frame = sig, frame
-
-
-__all__ = (
-    "DelayedSignal",
-    "SIGINT",
-)
diff --git a/src/tox/util/spinner.py b/src/tox/util/spinner.py
index 136b6560..e932b0b9 100644
--- a/src/tox/util/spinner.py
+++ b/src/tox/util/spinner.py
@@ -5,7 +5,7 @@ import threading
 import time
 from collections import OrderedDict
 from types import TracebackType
-from typing import IO, Dict, Optional, Sequence, Type
+from typing import IO, Dict, Optional, Sequence, Type, TypeVar
 
 from colorama import Fore
 
@@ -29,6 +29,10 @@ def _file_support_encoding(chars: Sequence[str], file: IO[str]) -> bool:
     return False
 
 
+T = TypeVar("T", bound="Spinner")
+MISS_DURATION = 0.01
+
+
 class Spinner:
     CLEAR_LINE = "\033[K"
     max_width = 120
@@ -36,7 +40,12 @@ class Spinner:
     ASCII_FRAMES = ["|", "-", "+", "x", "*"]
 
     def __init__(
-        self, enabled: bool = True, refresh_rate: float = 0.1, colored: bool = True, stream: Optional[IO[str]] = None
+        self,
+        enabled: bool = True,
+        refresh_rate: float = 0.1,
+        colored: bool = True,
+        stream: Optional[IO[str]] = None,
+        total: Optional[int] = None,
     ) -> None:
         self.is_colored = colored
         self.refresh_rate = refresh_rate
@@ -44,6 +53,7 @@ class Spinner:
         stream = sys.stdout if stream is None else stream
         self.frames = self.UNICODE_FRAMES if _file_support_encoding(self.UNICODE_FRAMES, stream) else self.ASCII_FRAMES
         self.stream = stream
+        self.total = total
 
         self._envs: Dict[str, float] = OrderedDict()
         self._frame_index = 0
@@ -70,12 +80,13 @@ class Spinner:
         frame = self.frames[self._frame_index]
         self._frame_index += 1
         self._frame_index %= len(self.frames)
-        text_frame = f"[{len(self._envs)}] {' | '.join(self._envs)}"
+        total = f"/{self.total}" if self.total is not None else ""
+        text_frame = f"[{len(self._envs)}{total}] {' | '.join(self._envs)}"
         if len(text_frame) > self.max_width - 1:
             text_frame = "{}...".format(text_frame[: self.max_width - 1 - 3])
         return "{} {}".format(*[(frame, text_frame)][0])
 
-    def __enter__(self) -> "Spinner":
+    def __enter__(self: T) -> T:
         if self.enabled:
             self.disable_cursor()
         self.render_frame()
@@ -114,11 +125,11 @@ class Spinner:
         self.finalize(key, "SKIP ⚠", Fore.YELLOW)
 
     def finalize(self, key: str, status: str, color: int) -> None:
-        start_at = self._envs[key]
-        del self._envs[key]
+        start_at = self._envs.pop(key, None)
         if self.enabled:
             self.clear()
-        base = f"{key}: {status} in {td_human_readable(time.monotonic() - start_at)}"
+        duration = MISS_DURATION if start_at is None else time.monotonic() - start_at
+        base = f"{key}: {status} in {td_human_readable(duration)}"
         if self.is_colored:
             base = f"{color}{base}{Fore.RESET}"
         base += os.linesep
diff --git a/src/tox/util/threading.py b/src/tox/util/threading.py
new file mode 100644
index 00000000..0e2db279
--- /dev/null
+++ b/src/tox/util/threading.py
@@ -0,0 +1,15 @@
+from threading import Lock
+
+
+class AtomicCounter:
+    def __init__(self) -> None:
+        self.value: int = 0
+        self._lock = Lock()
+
+    def increment(self) -> None:
+        with self._lock:
+            self.value += 1
+
+    def decrement(self) -> None:
+        with self._lock:
+            self.value -= 1