From 8736549a48c8467045ea2a56edddc9d4b17a4546 Mon Sep 17 00:00:00 2001 From: Masen Furer Date: Wed, 25 Jan 2023 11:24:56 -0800 Subject: Enforce constraints during install_package_deps (#2888) Fix https://github.com/tox-dev/tox/issues/2386 --- docs/changelog/2386.feature.rst | 12 +++ docs/config.rst | 19 ++++ docs/faq.rst | 25 ++--- src/tox/pytest.py | 1 + src/tox/tox_env/python/pip/pip_install.py | 51 +++++++++- tests/tox_env/python/pip/test_pip_install.py | 138 ++++++++++++++++++++++++++- 6 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 docs/changelog/2386.feature.rst diff --git a/docs/changelog/2386.feature.rst b/docs/changelog/2386.feature.rst new file mode 100644 index 00000000..2c631cb3 --- /dev/null +++ b/docs/changelog/2386.feature.rst @@ -0,0 +1,12 @@ +Test environments now recognize boolean config keys ``constrain_package_deps`` (default=true) and ``use_frozen_constraints`` (default=false), +which control how tox generates and applies constraints files when performing ``install_package_deps``. + +If ``constrain_package_deps`` is true (default), then tox will write out ``{env_dir}{/}constraints.txt`` and pass it to +``pip`` during ``install_package_deps``. If ``use_frozen_constraints`` is false (default), the constraints will be taken +from the specifications listed under ``deps`` (and inside any requirements or constraints file referenced in ``deps``). +Otherwise, ``list_dependencies_command`` (``pip freeze``) is used to enumerate exact package specifications which will +be written to the constraints file. + +In previous releases, conflicting package dependencies would silently override the ``deps`` named in the configuration, +resulting in test runs against unexpected dependency versions, particularly when using tox factors to explicitly test +with different versions of dependencies - by :user:`masenf`. diff --git a/docs/config.rst b/docs/config.rst index e80da066..7b702f3f 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -755,6 +755,25 @@ Pip installer latest available pre-release of any dependencies without a specified version. If ``false``, pip will only install final releases of unpinned dependencies. +.. conf:: + :keys: constrain_package_deps + :default: true + :version_added: 4.4.0 + + If ``constrain_package_deps`` is true, then tox will create and use ``{env_dir}{/}constraints.txt`` when installing + package dependnecies during ``install_package_deps`` stage. When this value is set to false, any conflicting package + dependencies will override explicit dependencies and constraints passed to ``deps``. + +.. conf:: + :keys: use_frozen_constraints + :default: false + :version_added: 4.4.0 + + When ``use_frozen_constraints`` is true, then tox will use the ``list_dependencies_command`` to enumerate package + versions in order to create ``{env_dir}{/}constraints.txt``. Otherwise the package specifications explicitly listed under + ``deps`` (or in requirements / constraints files referenced in ``deps``) will be used as the constraints. If + ``constrain_package_deps`` is false, then this setting has no effect. + User configuration ------------------ diff --git a/docs/faq.rst b/docs/faq.rst index aa47ff0c..9c948d8c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -124,21 +124,16 @@ install. While creating a test environment tox will invoke pip multiple times, i 1. install the dependencies of the package. 2. install the package itself. -Some solutions and their drawbacks: - -- specify the constraint files within :ref:`deps` (these constraints will not be applied when installing package - dependencies), -- use ``PIP_CONSTRAINT`` inside :ref:`set_env` (tox will not know about the content of the constraint file and such - will not trigger a rebuild of the environment when its content changes), -- specify the constraint file by extending the :ref:`install_command` as in the following example - (tox will not know about the content of the constraint file and such will not trigger a rebuild of the environment - when its content changes). - -.. code-block:: ini - - [testenv:py39] - install_command = python -m pip install {opts} {packages} -c constraints.txt - extras = test +Starting in tox 4.4.0, ``{env_dir}{/}constraints.txt`` is generated by default during ``install_deps`` based on the +package specifications listed under ``deps``. These constraints are subsequently passed to pip during the +``install_package_deps`` stage, causing an error to be raised when the package dependencies conflict with the test +environment dependencies. For stronger guarantees, set ``use_frozen_constraints = true`` in the test environment to +generate the constraints file based on the exact versions enumerated by the ``list_dependencies_command`` (``pip +freeze``). When using frozen constraints, if the package deps are incompatible with any previously installed +dependency, an error will be raised. + +Ensure that ``constrain_package_deps = true`` is set in the test environment in order to use the constraints file +generated by processing the ``deps`` section when performing ``package_deps``. Note constraint files are a subset of requirement files. Therefore, it's valid to pass a constraint file wherever you can specify a requirement file. diff --git a/src/tox/pytest.py b/src/tox/pytest.py index ae211252..fb83721c 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -525,6 +525,7 @@ __all__ = ( "LogCaptureFixture", "TempPathFactory", "MonkeyPatch", + "SubRequest", "ToxRunOutcome", "ToxProject", "ToxProjectCreator", diff --git a/src/tox/tox_env/python/pip/pip_install.py b/src/tox/tox_env/python/pip/pip_install.py index 2136e862..fcd445a5 100644 --- a/src/tox/tox_env/python/pip/pip_install.py +++ b/src/tox/tox_env/python/pip/pip_install.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging from collections import defaultdict +from pathlib import Path from typing import Any, Callable, Sequence from packaging.requirements import Requirement @@ -38,6 +39,18 @@ class Pip(Installer[Python]): post_process=self.post_process_install_command, desc="command used to install packages", ) + self._env.conf.add_config( + keys=["constrain_package_deps"], + of_type=bool, + default=True, + desc="If true, apply constraints during install_package_deps.", + ) + self._env.conf.add_config( + keys=["use_frozen_constraints"], + of_type=bool, + default=False, + desc="Use the exact versions of installed deps as constraints, otherwise use the listed deps.", + ) if self._with_list_deps: # pragma: no branch self._env.conf.add_config( keys=["list_dependencies_command"], @@ -81,6 +94,17 @@ class Pip(Installer[Python]): logging.warning(f"pip cannot install {arguments!r}") raise SystemExit(1) + def constraints_file(self) -> Path: + return Path(self._env.env_dir) / "constraints.txt" + + @property + def constrain_package_deps(self) -> bool: + return bool(self._env.conf["constrain_package_deps"]) + + @property + def use_frozen_constraints(self) -> bool: + return bool(self._env.conf["use_frozen_constraints"]) + def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None: try: new_options, new_reqs = arguments.unroll() @@ -90,7 +114,16 @@ class Pip(Installer[Python]): new_constraints: list[str] = [] for req in new_reqs: (new_constraints if req.startswith("-c ") else new_requirements).append(req) - new = {"options": new_options, "requirements": new_requirements, "constraints": new_constraints} + constraint_options = { + "constrain_package_deps": self.constrain_package_deps, + "use_frozen_constraints": self.use_frozen_constraints, + } + new = { + "options": new_options, + "requirements": new_requirements, + "constraints": new_constraints, + "constraint_options": constraint_options, + } # if option or constraint change in any way recreate, if the requirements change only if some are removed with self._env.cache.compare(new, section, of_type) as (eq, old): if not eq: # pragma: no branch @@ -100,9 +133,16 @@ class Pip(Installer[Python]): missing_requirement = set(old["requirements"]) - set(new_requirements) if missing_requirement: raise Recreate(f"requirements removed: {' '.join(missing_requirement)}") + old_constraint_options = old.get("constraint_options") + if old_constraint_options != constraint_options: + msg = f"constraint options changed: old={old_constraint_options} new={constraint_options}" + raise Recreate(msg) args = arguments.as_root_args if args: # pragma: no branch self._execute_installer(args, of_type) + if self.constrain_package_deps and not self.use_frozen_constraints: + combined_constraints = new_requirements + [c.lstrip("-c ") for c in new_constraints] + self.constraints_file().write_text("\n".join(combined_constraints)) @staticmethod def _recreate_if_diff(of_type: str, new_opts: list[str], old_opts: list[str], fmt: Callable[[str], str]) -> None: @@ -155,10 +195,19 @@ class Pip(Installer[Python]): self._execute_installer(install_args, of_type) def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None: + if of_type == "package_deps" and self.constrain_package_deps: + constraints_file = self.constraints_file() + if constraints_file.exists(): + deps = [*deps, f"-c{constraints_file}"] + cmd = self.build_install_cmd(deps) outcome = self._env.execute(cmd, stdin=StdinSource.OFF, run_id=f"install_{of_type}") outcome.assert_success() + if of_type == "deps" and self.constrain_package_deps and self.use_frozen_constraints: + # freeze installed deps for use as constraints + self.constraints_file().write_text("\n".join(self.installed())) + def build_install_cmd(self, args: Sequence[str]) -> list[str]: try: cmd: Command = self._env.conf["install_command"] diff --git a/tests/tox_env/python/pip/test_pip_install.py b/tests/tox_env/python/pip/test_pip_install.py index 8fb70a5f..d6bcdd3c 100644 --- a/tests/tox_env/python/pip/test_pip_install.py +++ b/tests/tox_env/python/pip/test_pip_install.py @@ -8,7 +8,7 @@ from unittest.mock import Mock import pytest from packaging.requirements import Requirement -from tox.pytest import CaptureFixture, ToxProjectCreator +from tox.pytest import CaptureFixture, SubRequest, ToxProject, ToxProjectCreator from tox.tox_env.errors import Fail @@ -270,3 +270,139 @@ def test_pip_install_constraint_file_new(tox_project: ToxProjectCreator) -> None assert "py: recreate env because changed constraint(s) added a" in result_second.out, result_second.out assert execute_calls.call_count == 1 assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", "a", "-c", "c.txt"] + + +@pytest.fixture(params=[True, False]) +def constrain_package_deps(request: SubRequest) -> bool: + return bool(request.param) + + +@pytest.fixture(params=[True, False]) +def use_frozen_constraints(request: SubRequest) -> bool: + return bool(request.param) + + +@pytest.fixture( + params=[ + "explicit", + "requirements", + "constraints", + "explicit+requirements", + "requirements_indirect", + "requirements_constraints_indirect", + ], +) +def constrained_mock_project( + request: SubRequest, + tox_project: ToxProjectCreator, + demo_pkg_inline: Path, + constrain_package_deps: bool, + use_frozen_constraints: bool, +) -> tuple[ToxProject, list[str]]: + toml = (demo_pkg_inline / "pyproject.toml").read_text() + files = { + "pyproject.toml": toml.replace("requires = []", 'requires = ["setuptools"]') + + '\n[project]\nname = "demo"\nversion = "0.1"\ndependencies = ["foo > 2"]', + "build.py": (demo_pkg_inline / "build.py").read_text(), + } + exp_constraints: list[str] = [] + requirement = "foo==1.2.3" + constraint = "foo<2" + if request.param == "explicit": + deps = requirement + exp_constraints.append(requirement) + elif request.param == "requirements": + files["requirements.txt"] = f"--pre\n{requirement}" + deps = "-rrequirements.txt" + exp_constraints.append(requirement) + elif request.param == "constraints": + files["constraints.txt"] = constraint + deps = "-cconstraints.txt" + exp_constraints.append(constraint) + elif request.param == "explicit+requirements": + files["requirements.txt"] = f"--pre\n{requirement}" + deps = "\n\t-rrequirements.txt\n\tfoo" + exp_constraints.extend(["foo", requirement]) + elif request.param == "requirements_indirect": + files["foo.requirements.txt"] = f"--pre\n{requirement}" + files["requirements.txt"] = "-r foo.requirements.txt" + deps = "-rrequirements.txt" + exp_constraints.append(requirement) + elif request.param == "requirements_constraints_indirect": + files["foo.requirements.txt"] = f"--pre\n{requirement}" + files["foo.constraints.txt"] = f"{constraint}" + files["requirements.txt"] = "-r foo.requirements.txt\n-c foo.constraints.txt" + deps = "-rrequirements.txt" + exp_constraints.extend([requirement, constraint]) + else: # pragma: no cover + pytest.fail(f"Missing case: {request.param}") + files["tox.ini"] = ( + "[testenv]\npackage=wheel\n" + f"constrain_package_deps = {constrain_package_deps}\n" + f"use_frozen_constraints = {use_frozen_constraints}\n" + f"deps = {deps}" + ) + return tox_project(files), exp_constraints if constrain_package_deps else [] + + +def test_constrain_package_deps( + constrained_mock_project: tuple[ToxProject, list[str]], + constrain_package_deps: bool, + use_frozen_constraints: bool, +) -> None: + proj, exp_constraints = constrained_mock_project + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result_first = proj.run("r") + result_first.assert_success() + exp_run_ids = ["install_deps"] + if constrain_package_deps and use_frozen_constraints: + exp_run_ids.append("freeze") + exp_run_ids.extend( + [ + "install_requires", + "_optional_hooks", + "get_requires_for_build_wheel", + "build_wheel", + "install_package_deps", + "install_package", + "_exit", + ], + ) + run_ids = [i[0][3].run_id for i in execute_calls.call_args_list] + assert run_ids == exp_run_ids + constraints_file = proj.path / ".tox" / "py" / "constraints.txt" + if constrain_package_deps: + constraints = constraints_file.read_text().splitlines() + for call in execute_calls.call_args_list: + if call[0][3].run_id == "install_package_deps": + assert f"-c{constraints_file}" in call[0][3].cmd + if use_frozen_constraints: + for c in exp_constraints: + # when using frozen constraints with this mock, the mock package does NOT + # actually end up in the constraints, so assert it's not there + assert c not in constraints + for c in constraints: + assert c.partition("==")[0] in ["pip", "setuptools", "wheel"] + else: + for c in constraints: + assert c in exp_constraints + for c in exp_constraints: + assert c in constraints + else: + assert not constraints_file.exists() + + +@pytest.mark.parametrize("conf_key", ["constrain_package_deps", "use_frozen_constraints"]) +def test_change_constraint_options_recreates(tox_project: ToxProjectCreator, conf_key: str) -> None: + tox_ini_content = "[testenv:py]\ndeps=a\nskip_install=true" + proj = tox_project({"tox.ini": f"{tox_ini_content}\n{conf_key} = true"}) + proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + + result = proj.run("r") + result.assert_success() + + (proj.path / "tox.ini").write_text(f"{tox_ini_content}\n{conf_key} = false") + result_second = proj.run("r") + result_second.assert_success() + assert "recreate env because constraint options changed" in result_second.out + assert conf_key in result_second.out -- cgit v1.2.1