Packaging inherits from pkgenv, deps and document tox 4 packaging changes (#2813)

Resolves https://github.com/tox-dev/tox/issues/2543

13 files changed, 166 insertions, 9 deletions

diff --git a/docs/changelog/2543.doc.rst b/docs/changelog/2543.doc.rst
new file mode 100644
index 00000000..59af6d2f
--- /dev/null
+++ b/docs/changelog/2543.doc.rst
@@ -0,0 +1 @@
+Document breaking changes with tox 4 and packaging environments - by :user:`gaborbernat`.
diff --git a/docs/changelog/2543.feature.rst b/docs/changelog/2543.feature.rst
new file mode 100644
index 00000000..c605a26d
--- /dev/null
+++ b/docs/changelog/2543.feature.rst
@@ -0,0 +1,3 @@
+Packaging environments now inherit from the ``pkgenv`` section, allowing to set all your packaging options in one place,
+and support the ``deps`` key to set additional dependencies that will be installed after ``pyprojec.toml`` static
+``requires`` but before backends dynamic requires - by :user:`gaborbernat`.
diff --git a/docs/upgrading.rst b/docs/upgrading.rst
index 459cfe4a..98b6c4e5 100644
--- a/docs/upgrading.rst
+++ b/docs/upgrading.rst
@@ -222,3 +222,96 @@ This is best avoided by updating to non-legacy usage:
 
     # or, equivalently...
     $ tox r -e list
+
+Packaging environments
+----------------------
+
+Isolated environment on by default
+++++++++++++++++++++++++++++++++++
+``tox`` now always uses an isolated build environment when building your projects package. The previous flag to enable
+this called ``isolated_build`` has been removed.
+
+Packaging configuration and inheritance
++++++++++++++++++++++++++++++++++++++++
+Isolated build environments are tox environments themselves and may be configured on their own. Their name is defined
+as follows:
+
+- For source distributions this environment will match a virtual environment with the same python interpreter as tox is
+  using. The name of this environment will by default ``.pkg`` (can be changed via :ref:`package_env` config on a per
+  test environment basis).
+- For wheels (including editable wheels as defined by :pep:`660`) their name will be ``.pkg-<impl><python_version>``, so
+  for example if you're building a wheel for a Python 3.10 environment the packaging environment will be
+  ``.pkg-cpython311``  (can be changed via :ref:`wheel_build_env` config on a per test environment basis).
+
+To change a packaging environments settings you can use:
+
+.. code-block:: ini
+
+    [testenv:.pkg]
+    pass_env =
+        PKG_CONFIG
+        PKG_CONFIG_PATH
+        PKG_CONFIG_SYSROOT_DIR
+
+    [testenv:.pkg-cpython311]
+    pass_env =
+        PKG_CONFIG
+        PKG_CONFIG_PATH
+        PKG_CONFIG_SYSROOT_DIR
+
+Packaging environments no longer inherit their settings from the ``testenv`` section, as this caused issues when
+some test environment settings conflicted with packaging setting. However starting with ``tox>=4.2`` all packaging
+environments inherit from the ``pkgenv`` section, allowing you to define packaging common packaging settings in one
+central place, while still allowing you to override it when needed on a per package environment basis:
+
+.. code-block:: ini
+
+    [pkgenv]
+    pass_env =
+        PKG_CONFIG
+        PKG_CONFIG_PATH
+        PKG_CONFIG_SYSROOT_DIR
+
+    [testenv:.pkg-cpython311]
+    pass_env =
+        {[pkgenv]pass_env}
+        IS_311 = yes
+
+    [testenv:magic]
+    package = sdist
+    pass_env = {[pkgenv]pass_env}  # sdist install builds wheel -> need packaging settings
+
+Note that specific packaging environments are defined under ``testenv:.pkg`` and **not** ``pkgenv:.pkg``, this is due
+backwards compatibility.
+
+Universal wheels
+++++++++++++++++
+If your project builds universal wheels you can avoid using multiple build environments for each targeted python by
+setting :ref:`wheel_build_env` to the same packaging environment via:
+
+.. code-block:: ini
+
+    [testenv]
+    package = wheel
+    wheel_build_env = .pkg
+
+Editable mode
++++++++++++++
+``tox`` now defaults to using editable wheels when develop mode is enabled and the build backend supports it,
+as defined by :pep:`660` by setting :ref:`package` to ``editable``. In case the backend does not support it, will
+fallback to :ref:`package` to ``editable-legacy``, and invoke pip with ``-e``. In the later case will also print a
+message to make this setting explicit in your configuration (explicit better than implicit):
+
+.. code-block:: ini
+
+    [testenv:dev]
+    package = editable-legacy
+
+If you want to use the new standardized method to achieve the editable install effect you should ensure your backend
+version is above the version this feature was added to it, for example for setuptools:
+
+.. code-block:: ini
+
+    [testenv:dev]
+    deps = setuptools>=64
+    package = editable
diff --git a/src/tox/config/main.py b/src/tox/config/main.py
index 38efb600..00aaaed8 100644
--- a/src/tox/config/main.py
+++ b/src/tox/config/main.py
@@ -152,10 +152,10 @@ class Config:
         :param loaders: loaders to use for this configuration (only used for creation)
         :return: the tox environments config
         """
-        section, base = self._src.get_tox_env_section(item)
+        section, base_test, base_pkg = self._src.get_tox_env_section(item)
         conf_set = self.get_section_config(
             section,
-            base=None if package else base,
+            base=base_pkg if package else base_test,
             of_type=EnvConfigSet,
             for_env=item,
             loaders=loaders,
@@ -163,7 +163,7 @@ class Config:
         return conf_set
 
     def clear_env(self, name: str) -> None:
-        section, _ = self._src.get_tox_env_section(name)
+        section, _, __ = self._src.get_tox_env_section(name)
         del self._key_to_conf_set[(section.key, name)]
 
 
diff --git a/src/tox/config/source/api.py b/src/tox/config/source/api.py
index 3b95d504..e762d479 100644
--- a/src/tox/config/source/api.py
+++ b/src/tox/config/source/api.py
@@ -98,7 +98,7 @@ class Source(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def get_tox_env_section(self, item: str) -> tuple[Section, list[str]]:
+    def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]:
         """:returns: the section for a tox environment"""
         raise NotImplementedError
 
diff --git a/src/tox/config/source/ini.py b/src/tox/config/source/ini.py
index f3e0c083..df6c2876 100644
--- a/src/tox/config/source/ini.py
+++ b/src/tox/config/source/ini.py
@@ -14,7 +14,7 @@ from ..loader.ini import IniLoader
 from ..loader.section import Section
 from ..sets import ConfigSet
 from .api import Source
-from .ini_section import CORE, TEST_ENV_PREFIX, IniSection
+from .ini_section import CORE, PKG_ENV_PREFIX, TEST_ENV_PREFIX, IniSection
 
 
 class IniSource(Source):
@@ -62,8 +62,8 @@ class IniSource(Source):
             if in_section.prefix is not None:  # no prefix specified, so this could imply our own prefix
                 yield IniSection(in_section.prefix, a_base)
 
-    def get_tox_env_section(self, item: str) -> tuple[Section, list[str]]:
-        return IniSection.test_env(item), [TEST_ENV_PREFIX]
+    def get_tox_env_section(self, item: str) -> tuple[Section, list[str], list[str]]:
+        return IniSection.test_env(item), [TEST_ENV_PREFIX], [PKG_ENV_PREFIX]
 
     def envs(self, core_config: ConfigSet) -> Iterator[str]:
         seen = set()
diff --git a/src/tox/config/source/ini_section.py b/src/tox/config/source/ini_section.py
index 33a71863..a155dc1f 100644
--- a/src/tox/config/source/ini_section.py
+++ b/src/tox/config/source/ini_section.py
@@ -20,10 +20,12 @@ class IniSection(Section):
 
 
 TEST_ENV_PREFIX = "testenv"
+PKG_ENV_PREFIX = "pkgenv"
 CORE = IniSection(None, "tox")
 
 __all__ = [
     "IniSection",
     "CORE",
     "TEST_ENV_PREFIX",
+    "PKG_ENV_PREFIX",
 ]
diff --git a/src/tox/tox_env/python/package.py b/src/tox/tox_env/python/package.py
index 910554a8..289bd774 100644
--- a/src/tox/tox_env/python/package.py
+++ b/src/tox/tox_env/python/package.py
@@ -5,7 +5,7 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generator, Iterator, Sequence, cast
+from typing import TYPE_CHECKING, Any, Generator, Iterator, List, Sequence, cast
 
 from packaging.requirements import Requirement
 
@@ -56,6 +56,7 @@ class PythonPackageToxEnv(Python, PackageToxEnv, ABC):
         """setup the tox environment"""
         super()._setup_env()
         self._install(self.requires(), PythonPackageToxEnv.__name__, "requires")
+        self._install(self.conf["deps"], PythonPackageToxEnv.__name__, "deps")
 
     @abstractmethod
     def requires(self) -> tuple[Requirement, ...] | PythonDeps:
@@ -63,6 +64,14 @@ class PythonPackageToxEnv(Python, PackageToxEnv, ABC):
 
     def register_run_env(self, run_env: RunToxEnv) -> Generator[tuple[str, str], PackageToxEnv, None]:
         yield from super().register_run_env(run_env)
+        if run_env.conf["package"] != "skip" and "deps" not in self.conf:
+            self.conf.add_config(
+                keys="deps",
+                of_type=List[Requirement],
+                default=[],
+                desc="Name of the python dependencies as specified by PEP-440",
+            )
+
         if (
             not isinstance(run_env, Python)
             or run_env.conf["package"] not in {"wheel", "editable"}
diff --git a/src/tox/util/ci.py b/src/tox/util/ci.py
index b65b2be6..30ba80ed 100644
--- a/src/tox/util/ci.py
+++ b/src/tox/util/ci.py
@@ -21,7 +21,12 @@ _ENV_VARS = {  # per https://adamj.eu/tech/2020/03/09/detect-if-your-tests-are-r
 
 def is_ci() -> bool:
     """:return: a flag indicating if running inside a CI env or not"""
-    return any(e in os.environ if v is None else os.environ.get(e) == v for e, v in _ENV_VARS.items())
+    for env_key, value in _ENV_VARS.items():
+        if env_key in os.environ if value is None else os.environ.get(env_key) == value:
+            if env_key == "TEAMCITY_VERSION" and os.environ.get(env_key) == "LOCAL":
+                continue
+            return True
+    return False
 
 
 __all__ = [
diff --git a/tests/session/cmd/test_show_config.py b/tests/session/cmd/test_show_config.py
index 646dc2bf..f0acb3d7 100644
--- a/tests/session/cmd/test_show_config.py
+++ b/tests/session/cmd/test_show_config.py
@@ -254,3 +254,20 @@ def test_show_config_matching_env_section(tox_project: ToxProjectCreator) -> Non
     outcome = project.run("c", "-e", "a,b", "-k", "deps")
     outcome.assert_success()
     assert outcome.out.count("c>=1") == 2, outcome.out
+
+
+def test_package_env_inherits_from_pkgenv(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None:
+    project = tox_project({"tox.ini": "[pkgenv]\npass_env = A, B\ndeps=C\n D"})
+    outcome = project.run("c", "--root", str(demo_pkg_inline), "-k", "deps", "pass_env", "-e", "py,.pkg")
+    outcome.assert_success()
+    exp = """
+    [testenv:.pkg]
+    deps =
+      C
+      D
+    pass_env =
+      A
+      B
+    """
+    exp = dedent(exp)
+    assert exp in outcome.out
diff --git a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py
index 839225b9..a8f6c9ed 100644
--- a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py
+++ b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py
@@ -194,3 +194,18 @@ def test_pyproject_no_build_editable_fallback(tox_project: ToxProjectCreator, de
     ]
     found_calls = [(i[0][0].conf.name, i[0][3].run_id) for i in execute_calls.call_args_list]
     assert found_calls == expected_calls
+
+
+@pytest.mark.parametrize("package", ["sdist", "wheel", "editable", "editable-legacy", "skip"])
+def test_project_package_with_deps(tox_project: ToxProjectCreator, demo_pkg_setuptools: Path, package: str) -> None:
+    ini = f"[testenv]\npackage={package}\n[pkgenv]\ndeps = A"
+    proj = tox_project({"tox.ini": ini}, base=demo_pkg_setuptools)
+    execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None)
+    result = proj.run("r", "--notest")
+    result.assert_success()
+    found_calls = [(i[0][0].conf.name, i[0][3].run_id) for i in execute_calls.call_args_list]
+    if package == "skip":
+        assert (".pkg", "install_deps") not in found_calls
+    else:
+        assert found_calls[0] == (".pkg", "install_requires")
+        assert found_calls[1] == (".pkg", "install_deps")
diff --git a/tests/util/test_ci.py b/tests/util/test_ci.py
index 5cfc9bb2..ef9a089d 100644
--- a/tests/util/test_ci.py
+++ b/tests/util/test_ci.py
@@ -54,3 +54,12 @@ def test_is_ci_not(monkeypatch: pytest.MonkeyPatch) -> None:
     for var in _ENV_VARS:
         monkeypatch.delenv(var, raising=False)
     assert not is_ci()
+
+
+def test_is_ci_not_teamcity_local(monkeypatch: pytest.MonkeyPatch) -> None:
+    # pycharm sets this
+    for var in _ENV_VARS:
+        monkeypatch.delenv(var, raising=False)
+
+    monkeypatch.setenv("TEAMCITY_VERSION", "LOCAL")
+    assert not is_ci()
diff --git a/whitelist.txt b/whitelist.txt
index ac48cb4b..c79f0c76 100644
--- a/whitelist.txt
+++ b/whitelist.txt
@@ -111,6 +111,7 @@ openpty
 ov
 pathname
 pep517
+pkgenv
 platformdirs
 pluggy
 pos
@@ -124,6 +125,7 @@ purelib
 py311
 py38
 py39
+pycharm
 pygments
 pypa
 pyproject
@@ -160,6 +162,7 @@ subparsers
 tcgetattr
 tcsanow
 tcsetattr
+teamcity
 termios
 termux
 testenv