Enforce constraints during install_package_deps (#2888)

Fix https://github.com/tox-dev/tox/issues/2386

6 files changed, 229 insertions, 17 deletions

diff --git a/docs/changelog/2386.feature.rst b/docs/changelog/2386.feature.rst
new file mode 100644
index 00000000..2c631cb3
--- /dev/null
+++ b/docs/changelog/2386.feature.rst
@@ -0,0 +1,12 @@
+Test environments now recognize boolean config keys ``constrain_package_deps`` (default=true) and ``use_frozen_constraints`` (default=false),
+which control how tox generates and applies constraints files when performing ``install_package_deps``.
+
+If ``constrain_package_deps`` is true (default), then tox will write out ``{env_dir}{/}constraints.txt`` and pass it to
+``pip`` during ``install_package_deps``. If ``use_frozen_constraints`` is false (default), the constraints will be taken
+from the specifications listed under ``deps`` (and inside any requirements or constraints file referenced in ``deps``).
+Otherwise, ``list_dependencies_command`` (``pip freeze``) is used to enumerate exact package specifications which will
+be written to the constraints file.
+
+In previous releases, conflicting package dependencies would silently override the ``deps`` named in the configuration,
+resulting in test runs against unexpected dependency versions, particularly when using tox factors to explicitly test
+with different versions of dependencies - by :user:`masenf`.
diff --git a/docs/config.rst b/docs/config.rst
index e80da066..7b702f3f 100644
--- a/docs/config.rst
+++ b/docs/config.rst
@@ -755,6 +755,25 @@ Pip installer
    latest available pre-release of any dependencies without a specified version. If ``false``, pip will only install
    final releases of unpinned dependencies.
 
+.. conf::
+   :keys: constrain_package_deps
+   :default: true
+   :version_added: 4.4.0
+
+   If ``constrain_package_deps`` is true, then tox will create and use ``{env_dir}{/}constraints.txt`` when installing
+   package dependnecies during ``install_package_deps`` stage. When this value is set to false, any conflicting package
+   dependencies will override explicit dependencies and constraints passed to ``deps``.
+
+.. conf::
+   :keys: use_frozen_constraints
+   :default: false
+   :version_added: 4.4.0
+
+   When ``use_frozen_constraints`` is true, then tox will use the ``list_dependencies_command`` to enumerate package
+   versions in order to create ``{env_dir}{/}constraints.txt``. Otherwise the package specifications explicitly listed under
+   ``deps`` (or in requirements / constraints files referenced in ``deps``) will be used as the constraints. If
+   ``constrain_package_deps`` is false, then this setting has no effect.
+
 User configuration
 ------------------
 
diff --git a/docs/faq.rst b/docs/faq.rst
index aa47ff0c..9c948d8c 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -124,21 +124,16 @@ install. While creating a test environment tox will invoke pip multiple times, i
    1. install the dependencies of the package.
    2. install the package itself.
 
-Some solutions and their drawbacks:
-
-- specify the constraint files within :ref:`deps` (these constraints will not be applied when installing package
-  dependencies),
-- use ``PIP_CONSTRAINT`` inside :ref:`set_env` (tox will not know about the content of the constraint file and such
-  will not trigger a rebuild of the environment when its content changes),
-- specify the constraint file by extending the :ref:`install_command` as in the following example
-  (tox will not know about the content of the constraint file and such will not trigger a rebuild of the environment
-  when its content changes).
-
-.. code-block:: ini
-
-    [testenv:py39]
-    install_command = python -m pip install {opts} {packages} -c constraints.txt
-    extras = test
+Starting in tox 4.4.0, ``{env_dir}{/}constraints.txt`` is generated by default during ``install_deps`` based on the
+package specifications listed under ``deps``. These constraints are subsequently passed to pip during the
+``install_package_deps`` stage, causing an error to be raised when the package dependencies conflict with the test
+environment dependencies. For stronger guarantees, set ``use_frozen_constraints = true`` in the test environment to
+generate the constraints file based on the exact versions enumerated by the ``list_dependencies_command`` (``pip
+freeze``).  When using frozen constraints, if the package deps are incompatible with any previously installed
+dependency, an error will be raised.
+
+Ensure that ``constrain_package_deps = true`` is set in the test environment in order to use the constraints file
+generated by processing the ``deps`` section when performing ``package_deps``.
 
 Note constraint files are a subset of requirement files. Therefore, it's valid to pass a constraint file wherever you
 can specify a requirement file.
diff --git a/src/tox/pytest.py b/src/tox/pytest.py
index ae211252..fb83721c 100644
--- a/src/tox/pytest.py
+++ b/src/tox/pytest.py
@@ -525,6 +525,7 @@ __all__ = (
     "LogCaptureFixture",
     "TempPathFactory",
     "MonkeyPatch",
+    "SubRequest",
     "ToxRunOutcome",
     "ToxProject",
     "ToxProjectCreator",
diff --git a/src/tox/tox_env/python/pip/pip_install.py b/src/tox/tox_env/python/pip/pip_install.py
index 2136e862..fcd445a5 100644
--- a/src/tox/tox_env/python/pip/pip_install.py
+++ b/src/tox/tox_env/python/pip/pip_install.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 from collections import defaultdict
+from pathlib import Path
 from typing import Any, Callable, Sequence
 
 from packaging.requirements import Requirement
@@ -38,6 +39,18 @@ class Pip(Installer[Python]):
             post_process=self.post_process_install_command,
             desc="command used to install packages",
         )
+        self._env.conf.add_config(
+            keys=["constrain_package_deps"],
+            of_type=bool,
+            default=True,
+            desc="If true, apply constraints during install_package_deps.",
+        )
+        self._env.conf.add_config(
+            keys=["use_frozen_constraints"],
+            of_type=bool,
+            default=False,
+            desc="Use the exact versions of installed deps as constraints, otherwise use the listed deps.",
+        )
         if self._with_list_deps:  # pragma: no branch
             self._env.conf.add_config(
                 keys=["list_dependencies_command"],
@@ -81,6 +94,17 @@ class Pip(Installer[Python]):
             logging.warning(f"pip cannot install {arguments!r}")
             raise SystemExit(1)
 
+    def constraints_file(self) -> Path:
+        return Path(self._env.env_dir) / "constraints.txt"
+
+    @property
+    def constrain_package_deps(self) -> bool:
+        return bool(self._env.conf["constrain_package_deps"])
+
+    @property
+    def use_frozen_constraints(self) -> bool:
+        return bool(self._env.conf["use_frozen_constraints"])
+
     def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None:
         try:
             new_options, new_reqs = arguments.unroll()
@@ -90,7 +114,16 @@ class Pip(Installer[Python]):
         new_constraints: list[str] = []
         for req in new_reqs:
             (new_constraints if req.startswith("-c ") else new_requirements).append(req)
-        new = {"options": new_options, "requirements": new_requirements, "constraints": new_constraints}
+        constraint_options = {
+            "constrain_package_deps": self.constrain_package_deps,
+            "use_frozen_constraints": self.use_frozen_constraints,
+        }
+        new = {
+            "options": new_options,
+            "requirements": new_requirements,
+            "constraints": new_constraints,
+            "constraint_options": constraint_options,
+        }
         # if option or constraint change in any way recreate, if the requirements change only if some are removed
         with self._env.cache.compare(new, section, of_type) as (eq, old):
             if not eq:  # pragma: no branch
@@ -100,9 +133,16 @@ class Pip(Installer[Python]):
                     missing_requirement = set(old["requirements"]) - set(new_requirements)
                     if missing_requirement:
                         raise Recreate(f"requirements removed: {' '.join(missing_requirement)}")
+                    old_constraint_options = old.get("constraint_options")
+                    if old_constraint_options != constraint_options:
+                        msg = f"constraint options changed: old={old_constraint_options} new={constraint_options}"
+                        raise Recreate(msg)
                 args = arguments.as_root_args
                 if args:  # pragma: no branch
                     self._execute_installer(args, of_type)
+                    if self.constrain_package_deps and not self.use_frozen_constraints:
+                        combined_constraints = new_requirements + [c.lstrip("-c ") for c in new_constraints]
+                        self.constraints_file().write_text("\n".join(combined_constraints))
 
     @staticmethod
     def _recreate_if_diff(of_type: str, new_opts: list[str], old_opts: list[str], fmt: Callable[[str], str]) -> None:
@@ -155,10 +195,19 @@ class Pip(Installer[Python]):
             self._execute_installer(install_args, of_type)
 
     def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None:
+        if of_type == "package_deps" and self.constrain_package_deps:
+            constraints_file = self.constraints_file()
+            if constraints_file.exists():
+                deps = [*deps, f"-c{constraints_file}"]
+
         cmd = self.build_install_cmd(deps)
         outcome = self._env.execute(cmd, stdin=StdinSource.OFF, run_id=f"install_{of_type}")
         outcome.assert_success()
 
+        if of_type == "deps" and self.constrain_package_deps and self.use_frozen_constraints:
+            # freeze installed deps for use as constraints
+            self.constraints_file().write_text("\n".join(self.installed()))
+
     def build_install_cmd(self, args: Sequence[str]) -> list[str]:
         try:
             cmd: Command = self._env.conf["install_command"]
diff --git a/tests/tox_env/python/pip/test_pip_install.py b/tests/tox_env/python/pip/test_pip_install.py
index 8fb70a5f..d6bcdd3c 100644
--- a/tests/tox_env/python/pip/test_pip_install.py
+++ b/tests/tox_env/python/pip/test_pip_install.py
@@ -8,7 +8,7 @@ from unittest.mock import Mock
 import pytest
 from packaging.requirements import Requirement
 
-from tox.pytest import CaptureFixture, ToxProjectCreator
+from tox.pytest import CaptureFixture, SubRequest, ToxProject, ToxProjectCreator
 from tox.tox_env.errors import Fail
 
 
@@ -270,3 +270,139 @@ def test_pip_install_constraint_file_new(tox_project: ToxProjectCreator) -> None
     assert "py: recreate env because changed constraint(s) added a" in result_second.out, result_second.out
     assert execute_calls.call_count == 1
     assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", "a", "-c", "c.txt"]
+
+
+@pytest.fixture(params=[True, False])
+def constrain_package_deps(request: SubRequest) -> bool:
+    return bool(request.param)
+
+
+@pytest.fixture(params=[True, False])
+def use_frozen_constraints(request: SubRequest) -> bool:
+    return bool(request.param)
+
+
+@pytest.fixture(
+    params=[
+        "explicit",
+        "requirements",
+        "constraints",
+        "explicit+requirements",
+        "requirements_indirect",
+        "requirements_constraints_indirect",
+    ],
+)
+def constrained_mock_project(
+    request: SubRequest,
+    tox_project: ToxProjectCreator,
+    demo_pkg_inline: Path,
+    constrain_package_deps: bool,
+    use_frozen_constraints: bool,
+) -> tuple[ToxProject, list[str]]:
+    toml = (demo_pkg_inline / "pyproject.toml").read_text()
+    files = {
+        "pyproject.toml": toml.replace("requires = []", 'requires = ["setuptools"]')
+        + '\n[project]\nname = "demo"\nversion = "0.1"\ndependencies = ["foo > 2"]',
+        "build.py": (demo_pkg_inline / "build.py").read_text(),
+    }
+    exp_constraints: list[str] = []
+    requirement = "foo==1.2.3"
+    constraint = "foo<2"
+    if request.param == "explicit":
+        deps = requirement
+        exp_constraints.append(requirement)
+    elif request.param == "requirements":
+        files["requirements.txt"] = f"--pre\n{requirement}"
+        deps = "-rrequirements.txt"
+        exp_constraints.append(requirement)
+    elif request.param == "constraints":
+        files["constraints.txt"] = constraint
+        deps = "-cconstraints.txt"
+        exp_constraints.append(constraint)
+    elif request.param == "explicit+requirements":
+        files["requirements.txt"] = f"--pre\n{requirement}"
+        deps = "\n\t-rrequirements.txt\n\tfoo"
+        exp_constraints.extend(["foo", requirement])
+    elif request.param == "requirements_indirect":
+        files["foo.requirements.txt"] = f"--pre\n{requirement}"
+        files["requirements.txt"] = "-r foo.requirements.txt"
+        deps = "-rrequirements.txt"
+        exp_constraints.append(requirement)
+    elif request.param == "requirements_constraints_indirect":
+        files["foo.requirements.txt"] = f"--pre\n{requirement}"
+        files["foo.constraints.txt"] = f"{constraint}"
+        files["requirements.txt"] = "-r foo.requirements.txt\n-c foo.constraints.txt"
+        deps = "-rrequirements.txt"
+        exp_constraints.extend([requirement, constraint])
+    else:  # pragma: no cover
+        pytest.fail(f"Missing case: {request.param}")
+    files["tox.ini"] = (
+        "[testenv]\npackage=wheel\n"
+        f"constrain_package_deps = {constrain_package_deps}\n"
+        f"use_frozen_constraints = {use_frozen_constraints}\n"
+        f"deps = {deps}"
+    )
+    return tox_project(files), exp_constraints if constrain_package_deps else []
+
+
+def test_constrain_package_deps(
+    constrained_mock_project: tuple[ToxProject, list[str]],
+    constrain_package_deps: bool,
+    use_frozen_constraints: bool,
+) -> None:
+    proj, exp_constraints = constrained_mock_project
+    execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)
+    result_first = proj.run("r")
+    result_first.assert_success()
+    exp_run_ids = ["install_deps"]
+    if constrain_package_deps and use_frozen_constraints:
+        exp_run_ids.append("freeze")
+    exp_run_ids.extend(
+        [
+            "install_requires",
+            "_optional_hooks",
+            "get_requires_for_build_wheel",
+            "build_wheel",
+            "install_package_deps",
+            "install_package",
+            "_exit",
+        ],
+    )
+    run_ids = [i[0][3].run_id for i in execute_calls.call_args_list]
+    assert run_ids == exp_run_ids
+    constraints_file = proj.path / ".tox" / "py" / "constraints.txt"
+    if constrain_package_deps:
+        constraints = constraints_file.read_text().splitlines()
+        for call in execute_calls.call_args_list:
+            if call[0][3].run_id == "install_package_deps":
+                assert f"-c{constraints_file}" in call[0][3].cmd
+        if use_frozen_constraints:
+            for c in exp_constraints:
+                # when using frozen constraints with this mock, the mock package does NOT
+                # actually end up in the constraints, so assert it's not there
+                assert c not in constraints
+            for c in constraints:
+                assert c.partition("==")[0] in ["pip", "setuptools", "wheel"]
+        else:
+            for c in constraints:
+                assert c in exp_constraints
+            for c in exp_constraints:
+                assert c in constraints
+    else:
+        assert not constraints_file.exists()
+
+
+@pytest.mark.parametrize("conf_key", ["constrain_package_deps", "use_frozen_constraints"])
+def test_change_constraint_options_recreates(tox_project: ToxProjectCreator, conf_key: str) -> None:
+    tox_ini_content = "[testenv:py]\ndeps=a\nskip_install=true"
+    proj = tox_project({"tox.ini": f"{tox_ini_content}\n{conf_key} = true"})
+    proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)
+
+    result = proj.run("r")
+    result.assert_success()
+
+    (proj.path / "tox.ini").write_text(f"{tox_ini_content}\n{conf_key} = false")
+    result_second = proj.run("r")
+    result_second.assert_success()
+    assert "recreate env because constraint options changed" in result_second.out
+    assert conf_key in result_second.out