Improve interaction between pyproject.toml metadata and discovery

3 files changed, 115 insertions, 73 deletions

diff --git a/setuptools/config/expand.py b/setuptools/config/expand.py
index 694476a0..94c9ee38 100644
--- a/setuptools/config/expand.py
+++ b/setuptools/config/expand.py
@@ -29,9 +29,12 @@ from typing import (
     Callable,
     Dict,
     Iterable,
+    Iterator,
     List,
+    Mapping,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast
 )
@@ -46,6 +49,8 @@ if TYPE_CHECKING:
 
 chain_iter = chain.from_iterable
 _Path = Union[str, os.PathLike]
+_K = TypeVar("_K")
+_V = TypeVar("_V", covariant=True)
 
 
 class StaticModule:
@@ -146,7 +151,7 @@ def _assert_local(filepath: _Path, root_dir: str):
 
 def read_attr(
     attr_desc: str,
-    package_dir: Optional[dict] = None,
+    package_dir: Optional[Mapping[str, str]] = None,
     root_dir: Optional[_Path] = None
 ):
     """Reads the value of an attribute from a module.
@@ -203,7 +208,7 @@ def _load_spec(spec: ModuleSpec, module_name: str) -> ModuleType:
 
 
 def _find_module(
-    module_name: str, package_dir: Optional[dict], root_dir: _Path
+    module_name: str, package_dir: Optional[Mapping[str, str]], root_dir: _Path
 ) -> Tuple[_Path, Optional[str], str]:
     """Given a module (that could normally be imported by ``module_name``
     after the build is complete), find the path to the parent directory where
@@ -238,7 +243,7 @@ def _find_module(
 
 def resolve_class(
     qualified_class_name: str,
-    package_dir: Optional[dict] = None,
+    package_dir: Optional[Mapping[str, str]] = None,
     root_dir: Optional[_Path] = None
 ) -> Callable:
     """Given a qualified class name, return the associated class object"""
@@ -254,7 +259,7 @@ def resolve_class(
 
 def cmdclass(
     values: Dict[str, str],
-    package_dir: Optional[dict] = None,
+    package_dir: Optional[Mapping[str, str]] = None,
     root_dir: Optional[_Path] = None
 ) -> Dict[str, Callable]:
     """Given a dictionary mapping command names to strings for qualified class
@@ -378,12 +383,10 @@ class EnsurePackagesDiscovered:
     """Some expand functions require all the packages to already be discovered before
     they run, e.g. :func:`read_attr`, :func:`resolve_class`, :func:`cmdclass`.
 
-    Therefore in some cases we will need to run autodiscovery during the parsing of the
-    configuration. However, it is better to postpone calling package discovery as much
-    as possible.
-
-    We should only run the discovery if absolutely necessary, otherwise we can miss
-    files that define important configuration (like ``package_dir``) are processed.
+    Therefore in some cases we will need to run autodiscovery during the evaluation of
+    the configuration. However, it is better to postpone calling package discovery as
+    much as possible, because some parameters can influence it (e.g. ``package_dir``),
+    and those might not have been processed yet.
     """
 
     def __init__(self, distribution: "Distribution"):
@@ -391,9 +394,10 @@ class EnsurePackagesDiscovered:
         self._called = False
 
     def __call__(self):
-        self._called = True
-        self._dist.set_defaults(name=False)  # Skip name since we are parsing metadata
-        return self._dist.package_dir
+        """Trigger the automatic package discovery, if it is still necessary."""
+        if not self._called:
+            self._called = True
+            self._dist.set_defaults(name=False)  # Skip name, we can still be parsing
 
     def __enter__(self):
         return self
@@ -401,3 +405,45 @@ class EnsurePackagesDiscovered:
     def __exit__(self, _exc_type, _exc_value, _traceback):
         if self._called:
             self._dist.set_defaults.analyse_name()  # Now we can set a default name
+
+    def _get_package_dir(self) -> Mapping[str, str]:
+        self()
+        return self._dist.package_dir
+
+    @property
+    def package_dir(self) -> Mapping[str, str]:
+        """Proxy to ``package_dir`` that may trigger auto-discovery when used."""
+        return LazyMappingProxy(self._get_package_dir)
+
+
+class LazyMappingProxy(Mapping[_K, _V]):
+    """Mapping proxy that delays resolving the target object, until really needed.
+
+    >>> def obtain_mapping():
+    ...     print("Running expensive function!")
+    ...     return {"key": "value", "other key": "other value"}
+    >>> mapping = LazyMappingProxy(obtain_mapping)
+    >>> mapping["key"]
+    Running expensive function!
+    'value'
+    >>> mapping["other key"]
+    'other value'
+    """
+
+    def __init__(self, obtain_mapping_value: Callable[[], Mapping[_K, _V]]):
+        self._obtain = obtain_mapping_value
+        self._value: Optional[Mapping[_K, _V]] = None
+
+    def _target(self) -> Mapping[_K, _V]:
+        if self._value is None:
+            self._value = self._obtain()
+        return self._value
+
+    def __getitem__(self, key: _K) -> _V:
+        return self._target()[key]
+
+    def __len__(self) -> int:
+        return len(self._target())
+
+    def __iter__(self) -> Iterator[_K]:
+        return iter(self._target())
diff --git a/setuptools/config/pyprojecttoml.py b/setuptools/config/pyprojecttoml.py
index 2b430787..7867cd52 100644
--- a/setuptools/config/pyprojecttoml.py
+++ b/setuptools/config/pyprojecttoml.py
@@ -4,7 +4,7 @@ import os
 import warnings
 from contextlib import contextmanager
 from functools import partial
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Mapping, Union
 
 from setuptools.errors import FileError, OptionError
 
@@ -137,84 +137,80 @@ def expand_configuration(
     root_dir = root_dir or os.getcwd()
     project_cfg = config.get("project", {})
     setuptools_cfg = config.get("tool", {}).get("setuptools", {})
+    silent = ignore_option_errors
 
-    # A distribution object is required for discovering the correct package_dir
-    dist, setuptools_cfg = _ensure_dist_and_package_dir(
-        dist, project_cfg, setuptools_cfg, root_dir
-    )
-
-    _expand_packages(setuptools_cfg, root_dir, ignore_option_errors)
+    _expand_packages(setuptools_cfg, root_dir, silent)
     _canonic_package_data(setuptools_cfg)
     _canonic_package_data(setuptools_cfg, "exclude-package-data")
 
-    with _expand.EnsurePackagesDiscovered(dist) as ensure_discovered:
-        _fill_discovered_attrs(dist, setuptools_cfg, ensure_discovered)
-        package_dir = setuptools_cfg["package-dir"]
+    # A distribution object is required for discovering the correct package_dir
+    dist = _ensure_dist(dist, project_cfg, root_dir)
 
-        process = partial(_process_field, ignore_option_errors=ignore_option_errors)
+    with _EnsurePackagesDiscovered(dist, setuptools_cfg) as ensure_discovered:
+        package_dir = ensure_discovered.package_dir
+        process = partial(_process_field, ignore_option_errors=silent)
         cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir)
         data_files = partial(_expand.canonic_data_files, root_dir=root_dir)
 
         process(setuptools_cfg, "data-files", data_files)
         process(setuptools_cfg, "cmdclass", cmdclass)
-        _expand_all_dynamic(project_cfg, setuptools_cfg, root_dir, ignore_option_errors)
+        _expand_all_dynamic(project_cfg, setuptools_cfg, package_dir, root_dir, silent)
 
     return config
 
 
-def _ensure_dist_and_package_dir(
-    dist: Optional["Distribution"],
-    project_cfg: dict,
-    setuptools_cfg: dict,
-    root_dir: _Path,
-) -> Tuple["Distribution", dict]:
+def _ensure_dist(
+    dist: Optional["Distribution"], project_cfg: dict, root_dir: _Path
+) -> "Distribution":
     from setuptools.dist import Distribution
 
     attrs = {"src_root": root_dir, "name": project_cfg.get("name", None)}
-    dist = dist or Distribution(attrs)
-
-    # dist and setuptools_cfg should use the same package_dir
-    if dist.package_dir is None:
-        dist.package_dir = setuptools_cfg.get("package-dir", {})
-    if setuptools_cfg.get("package-dir") is None:
-        setuptools_cfg["package-dir"] = dist.package_dir
-
-    return dist, setuptools_cfg
-
-
-def _fill_discovered_attrs(
-    dist: "Distribution",
-    setuptools_cfg: dict,
-    ensure_discovered: _expand.EnsurePackagesDiscovered,
-):
-    """When entering the context, the values of ``packages``, ``py_modules`` and
-    ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.
-    When existing the context, if these values are missing in ``setuptools_cfg``, they
-    will be copied from ``dist``.
-    """
-    package_dir = setuptools_cfg["package-dir"]
-    dist.package_dir = package_dir  # need to be the same object
-
-    # Set `py_modules` and `packages` in dist to short-circuit auto-discovery,
-    # but avoid overwriting empty lists purposefully set by users.
-    if isinstance(setuptools_cfg.get("py-modules"), list) and dist.py_modules is None:
-        dist.py_modules = setuptools_cfg["py-modules"]
-    if isinstance(setuptools_cfg.get("packages"), list) and dist.packages is None:
-        dist.packages = setuptools_cfg["packages"]
-
-    package_dir.update(ensure_discovered())
-
-    # If anything was discovered set them back, so they count in the final config.
-    setuptools_cfg.setdefault("packages", dist.packages)
-    setuptools_cfg.setdefault("py-modules", dist.py_modules)
+    return dist or Distribution(attrs)
+
+
+class _EnsurePackagesDiscovered(_expand.EnsurePackagesDiscovered):
+    def __init__(self, distribution: "Distribution", setuptools_cfg: dict):
+        super().__init__(distribution)
+        self._setuptools_cfg = setuptools_cfg
+
+    def __enter__(self):
+        """When entering the context, the values of ``packages``, ``py_modules`` and
+        ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.
+        """
+        dist, cfg = self._dist, self._setuptools_cfg
+        package_dir: Dict[str, str] = cfg.setdefault("package-dir", {})
+        package_dir.update(dist.package_dir or {})
+        dist.package_dir = package_dir  # needs to be the same object
+
+        # Set `py_modules` and `packages` in dist to short-circuit auto-discovery,
+        # but avoid overwriting empty lists purposefully set by users.
+        if dist.py_modules is None:
+            dist.py_modules = cfg.get("py-modules")
+        if dist.packages is None:
+            dist.packages = cfg.get("packages")
+
+        return super().__enter__()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """When exiting the context, if values of ``packages``, ``py_modules`` and
+        ``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``.
+        """
+        # If anything was discovered set them back, so they count in the final config.
+        self._setuptools_cfg.setdefault("packages", self._dist.packages)
+        self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules)
+        return super().__exit__(exc_type, exc_value, traceback)
 
 
 def _expand_all_dynamic(
-    project_cfg: dict, setuptools_cfg: dict, root_dir: _Path, ignore_option_errors: bool
+    project_cfg: dict,
+    setuptools_cfg: dict,
+    package_dir: Mapping[str, str],
+    root_dir: _Path,
+    ignore_option_errors: bool,
 ):
     silent = ignore_option_errors
     dynamic_cfg = setuptools_cfg.get("dynamic", {})
-    pkg_dir = setuptools_cfg["package-dir"]
+    pkg_dir = package_dir
     special = (
         "readme",
         "version",
@@ -251,7 +247,7 @@ def _expand_all_dynamic(
 def _expand_dynamic(
     dynamic_cfg: dict,
     field: str,
-    package_dir: dict,
+    package_dir: Mapping[str, str],
     root_dir: _Path,
     ignore_option_errors: bool,
 ):
@@ -296,7 +292,7 @@ def _expand_packages(setuptools_cfg: dict, root_dir: _Path, ignore_option_errors
     find = packages.get("find")
     if isinstance(find, dict):
         find["root_dir"] = root_dir
-        find["fill_package_dir"] = setuptools_cfg["package-dir"]
+        find["fill_package_dir"] = setuptools_cfg.setdefault("package-dir", {})
         with _ignore_errors(ignore_option_errors):
             setuptools_cfg["packages"] = _expand.find_packages(**find)
 
diff --git a/setuptools/config/setupcfg.py b/setuptools/config/setupcfg.py
index 36460d95..5ecf6269 100644
--- a/setuptools/config/setupcfg.py
+++ b/setuptools/config/setupcfg.py
@@ -368,7 +368,7 @@ class ConfigHandler(Generic[Target]):
         attr_desc = value.replace(attr_directive, '')
 
         # Make sure package_dir is populated correctly, so `attr:` directives can work
-        package_dir.update(self.ensure_discovered())
+        package_dir.update(self.ensure_discovered.package_dir)
         return expand.read_attr(attr_desc, package_dir, root_dir)
 
     @classmethod
@@ -596,7 +596,7 @@ class ConfigOptionsHandler(ConfigHandler["Distribution"]):
         }
 
     def _parse_cmdclass(self, value):
-        package_dir = self.ensure_discovered()
+        package_dir = self.ensure_discovered.package_dir
         return expand.cmdclass(self._parse_dict(value), package_dir, self.root_dir)
 
     def _parse_packages(self, value):