Support recursive extras defined in pyproject.toml (#2905)

* test_package_pyproject: recursive extras

Add regression test for issue #2904

* test_package_pyproject: when project deps has a self-referential extra

the project depends on an extra defined within itself

* Support recursive extras defined in pyproject.toml

Expand extras that reference an extra of the same package name to respect local
changes to package metadata.

Fix #2904

4 files changed, 74 insertions, 6 deletions

diff --git a/docs/changelog/2904.bugfix.rst b/docs/changelog/2904.bugfix.rst
new file mode 100644
index 00000000..39628f2a
--- /dev/null
+++ b/docs/changelog/2904.bugfix.rst
@@ -0,0 +1,3 @@
+Tox will now expand self-referential extras discovered in package deps to respect local modifications to package
+metadata. This allows a package extra to explicitly depend on another package extra, which previously only worked with
+non-static metadata - by :user:`masenf`.
diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py
index 77508732..2e3e61b1 100644
--- a/src/tox/tox_env/python/virtual_env/package/pyproject.py
+++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py
@@ -33,7 +33,7 @@ from tox.tox_env.runner import RunToxEnv
 from tox.util.file_view import create_session_view
 
 from ..api import VirtualEnv
-from .util import dependencies_with_extras
+from .util import dependencies_with_extras, dependencies_with_extras_from_markers
 
 if sys.version_info >= (3, 8):  # pragma: no cover (py38+)
     from importlib.metadata import Distribution, PathDistribution
@@ -253,11 +253,17 @@ class Pep517VirtualEnvPackager(PythonPackageToxEnv, VirtualEnv):
             if dynamic == "dependencies" or (extras and dynamic == "optional-dependencies"):
                 return None  # if any dependencies are dynamic we can just calculate all dynamically
 
-        deps: list[Requirement] = [Requirement(i) for i in project.get("dependencies", [])]
+        deps_with_markers: list[tuple[Requirement, set[str | None]]] = [
+            (Requirement(i), {None}) for i in project.get("dependencies", [])
+        ]
         optional_deps = project.get("optional-dependencies", {})
-        for extra in extras:
-            deps.extend(Requirement(i) for i in optional_deps.get(extra, []))
-        return deps
+        for extra, reqs in optional_deps.items():
+            deps_with_markers.extend((Requirement(req), {extra}) for req in (reqs or []))
+        return dependencies_with_extras_from_markers(
+            deps_with_markers=deps_with_markers,
+            extras=extras,
+            package_name=project.get("name", "."),
+        )
 
     def _load_deps_from_built_metadata(self, for_env: EnvConfigSet) -> list[Requirement]:
         # dependencies might depend on the python environment we're running in => if we build a wheel use that env
diff --git a/src/tox/tox_env/python/virtual_env/package/util.py b/src/tox/tox_env/python/virtual_env/package/util.py
index 948de48c..603f93ed 100644
--- a/src/tox/tox_env/python/virtual_env/package/util.py
+++ b/src/tox/tox_env/python/virtual_env/package/util.py
@@ -8,7 +8,14 @@ from packaging.requirements import Requirement
 
 
 def dependencies_with_extras(deps: list[Requirement], extras: set[str], package_name: str) -> list[Requirement]:
-    deps_with_markers = extract_extra_markers(deps)
+    return dependencies_with_extras_from_markers(extract_extra_markers(deps), extras, package_name)
+
+
+def dependencies_with_extras_from_markers(
+    deps_with_markers: list[tuple[Requirement, set[str | None]]],
+    extras: set[str],
+    package_name: str,
+) -> list[Requirement]:
     result: list[Requirement] = []
     found: set[str] = set()
     todo: set[str | None] = extras | {None}
diff --git a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py
index a8f6c9ed..0d6160a5 100644
--- a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py
+++ b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from textwrap import dedent
 
 import pytest
 
@@ -92,6 +93,57 @@ def test_package_root_via_testenv(tox_project: ToxProjectCreator, demo_pkg_inlin
             ["A"],
             id="deps_with_dynamic_optional_no_extra",
         ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                dependencies=['foo[alpha]']
+                optional-dependencies.alpha=['A']""",
+            ),
+            "",
+            ["A"],
+            id="deps_reference_extra",
+        ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                dependencies=['A']
+                optional-dependencies.alpha=['B']
+                optional-dependencies.beta=['foo[alpha]']""",
+            ),
+            "beta",
+            ["A", "B"],
+            id="deps_with_recursive_extra",
+        ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                dependencies=['A']
+                optional-dependencies.alpha=['B']
+                optional-dependencies.beta=['foo[alpha]']
+                optional-dependencies.delta=['foo[beta]', 'D']""",
+            ),
+            "delta",
+            ["A", "B", "D"],
+            id="deps_with_two_recursive_extra",
+        ),
+        pytest.param(
+            dedent(
+                """
+                [project]
+                name='foo'
+                optional-dependencies.alpha=['foo[beta]', 'A']
+                optional-dependencies.beta=['foo[alpha]', 'B']""",
+            ),
+            "alpha",
+            ["A", "B"],
+            id="deps_with_circular_recursive_extra",
+        ),
     ],
 )
 def test_pyproject_deps_from_static(