diff options
author | Bernát Gábor <gaborjbernat@gmail.com> | 2022-12-05 07:48:34 -0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-12-05 07:48:34 -0800 |
commit | 4c77457137993c43da7a74aab49943283183f3e1 (patch) | |
tree | 83d83b893027505ee17cfac51bdd8bf3e6f864aa | |
parent | 3d50713bab1d5ac89ac7dd56303f0c63a8621dc7 (diff) | |
download | tox-git-4c77457137993c43da7a74aab49943283183f3e1.tar.gz |
Lock parallel package operations (#2593)
-rw-r--r-- | .pre-commit-config.yaml | 2 | ||||
-rw-r--r-- | docs/changelog/2594.bugfix.rst | 2 | ||||
-rw-r--r-- | pyproject.toml | 2 | ||||
-rw-r--r-- | src/tox/tox_env/api.py | 2 | ||||
-rw-r--r-- | src/tox/tox_env/package.py | 22 | ||||
-rw-r--r-- | src/tox/util/path.py | 4 |
6 files changed, 25 insertions, 9 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a8e8e0b..9946d38d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,7 +59,7 @@ repos: - flake8-unused-arguments==0.0.12 - flake8-noqa==1.3 - pep8-naming==0.13.2 - - flake8-pyproject==1.2.1 + - flake8-pyproject==1.2.2 - repo: https://github.com/pre-commit/mirrors-prettier rev: "v2.7.1" hooks: diff --git a/docs/changelog/2594.bugfix.rst b/docs/changelog/2594.bugfix.rst new file mode 100644 index 00000000..b36a6432 --- /dev/null +++ b/docs/changelog/2594.bugfix.rst @@ -0,0 +1,2 @@ +Ensure that two parallel tox instance invocations on different tox environment targets will work by holding a file lock +onto the packaging operations (e.g., in bash ``tox4 r -e py311 &; tox4 r -e py310``) - by :user:`gaborbernat`. diff --git a/pyproject.toml b/pyproject.toml index 9505d776..7589432e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pyproject-api>=1.2.1", 'tomli>=2.0.1; python_version < "3.11"', "virtualenv>=20.17", + "filelock>=3.8.1", 'importlib-metadata>=5.1; python_version < "3.8"', 'typing-extensions>=4.4; python_version < "3.8"', ] @@ -49,7 +50,6 @@ optional-dependencies.testing = [ "devpi-process>=0.3", "diff-cover>=7.2", "distlib>=0.3.6", - "filelock>=3.8", "flaky>=3.7", "hatch-vcs>=0.2", "hatchling>=1.11.1", diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py index 55b2cb4a..43a039e6 100644 --- a/src/tox/tox_env/api.py +++ b/src/tox/tox_env/api.py @@ -290,7 +290,7 @@ class ToxEnv(ABC): env_dir = self.env_dir if env_dir.exists(): LOGGER.warning("remove tox env folder %s", env_dir) - ensure_empty_dir(env_dir) + ensure_empty_dir(env_dir, except_filename="file.lock") self._log_id = 0 # we deleted logs, so start over counter self.cache.reset() self._run_state.update({"setup": False, "clean": True}) diff --git a/src/tox/tox_env/package.py b/src/tox/tox_env/package.py index cbdcbadc..5dcc680c 100644 --- a/src/tox/tox_env/package.py +++ b/src/tox/tox_env/package.py @@ -9,6 +9,8 @@ from threading import RLock from types import MethodType from typing import TYPE_CHECKING, Any, Callable, Generator, Iterator, cast +from filelock import FileLock + from tox.config.main import Config from tox.config.sets import EnvConfigSet @@ -31,17 +33,24 @@ class PathPackage(Package): return str(self.path) -def _lock_method(lock: RLock, meth: Callable[..., Any]) -> Callable[..., Any]: +def _lock_method(thread_lock: RLock, file_lock: FileLock | None, meth: Callable[..., Any]) -> Callable[..., Any]: def _func(*args: Any, **kwargs: Any) -> Any: - with lock: - return meth(*args, **kwargs) + with thread_lock: + if file_lock is not None and file_lock.is_locked is False: # file_lock is to lock from other tox processes + file_lock.acquire() + try: + return meth(*args, **kwargs) + finally: + if file_lock is not None: + file_lock.release() return _func class PackageToxEnv(ToxEnv, ABC): def __init__(self, create_args: ToxEnvCreateArgs) -> None: - self._lock = RLock() + self._thread_lock = RLock() + self._file_lock: FileLock | None = None super().__init__(create_args) self._envs: set[str] = set() @@ -49,11 +58,14 @@ class PackageToxEnv(ToxEnv, ABC): # the packaging class might be used by multiple environments in parallel, hold a lock for operations on it obj = object.__getattribute__(self, name) if isinstance(obj, MethodType): - obj = _lock_method(self._lock, obj) + obj = _lock_method(self._thread_lock, self._file_lock, obj) return obj def register_config(self) -> None: super().register_config() + file_lock_path: Path = self.conf["env_dir"] / "file.lock" + self._file_lock = FileLock(file_lock_path) + file_lock_path.parent.mkdir(parents=True, exist_ok=True) self.core.add_config( keys=["package_root", "setupdir"], of_type=Path, diff --git a/src/tox/util/path.py b/src/tox/util/path.py index b4a140bc..120bfa15 100644 --- a/src/tox/util/path.py +++ b/src/tox/util/path.py @@ -4,10 +4,12 @@ from pathlib import Path from shutil import rmtree -def ensure_empty_dir(path: Path) -> None: +def ensure_empty_dir(path: Path, except_filename: str | None = None) -> None: if path.exists(): if path.is_dir(): for sub_path in path.iterdir(): + if sub_path.name == except_filename: + continue if sub_path.is_dir(): rmtree(sub_path, ignore_errors=True) else: |