Support for environment files in set_env (#2223)

8 files changed, 119 insertions, 39 deletions

diff --git a/docs/changelog/1938.feature.rst b/docs/changelog/1938.feature.rst
new file mode 100644
index 00000000..76a4c4e6
--- /dev/null
+++ b/docs/changelog/1938.feature.rst
@@ -0,0 +1 @@
+Support for environment files within the :ref:`set_env` configuration via the ``file|`` prefix - by :user:`gaborbernat`.
diff --git a/docs/config.rst b/docs/config.rst
index fcb758c4..eb8cbca7 100644
--- a/docs/config.rst
+++ b/docs/config.rst
@@ -270,7 +270,17 @@ Base options
 .. conf::
    :keys: set_env, setenv
 
-   A dictionary of environment variables to set when running commands in the tox environment.
+   A dictionary of environment variables to set when running commands in the tox environment. Lines starting with a
+   ``file|`` prefix define the location of environment file.
+
+    .. note::
+
+       Environment files are processed using the following rules:
+
+       - blank lines are ignored,
+       - lines starting with the ``#`` character are ignored,
+       - each line is in KEY=VALUE format; both the key and the value are stripped,
+       - there is no special handling of quotation marks, they are part of the key or value.
 
 .. conf::
    :keys: parallel_show_output
diff --git a/src/tox/config/loader/ini/__init__.py b/src/tox/config/loader/ini/__init__.py
index 34aa18a1..c991a8be 100644
--- a/src/tox/config/loader/ini/__init__.py
+++ b/src/tox/config/loader/ini/__init__.py
@@ -81,8 +81,7 @@ class IniLoader(StrConvert, Loader[str]):
         yield raw
         if delay_replace:
             converted = future.result()
-            if hasattr(converted, "replacer"):  # pragma: no branch
-                converted.replacer = replacer  # type: ignore[attr-defined]
+            converted.use_replacer(replacer, args)  # type: ignore[attr-defined] # this can be only set_env that has it
 
     def found_keys(self) -> Set[str]:
         return set(self._section_proxy.keys())
diff --git a/src/tox/config/set_env.py b/src/tox/config/set_env.py
index 33cfae45..ef4d5362 100644
--- a/src/tox/config/set_env.py
+++ b/src/tox/config/set_env.py
@@ -1,34 +1,60 @@
+from pathlib import Path
 from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple
 
 from tox.config.loader.api import ConfigLoadArgs
+from tox.tox_env.errors import Fail
 
 Replacer = Callable[[str, ConfigLoadArgs], str]
 
 
 class SetEnv:
-    def __init__(self, raw: str, name: str, env_name: Optional[str]) -> None:
-        self.replacer: Replacer = lambda s, c: s
-        self._later: List[str] = []
-        self._raw: Dict[str, str] = {}
-        self._name, self._env_name = name, env_name
+    def __init__(self, raw: str, name: str, env_name: Optional[str], root: Path) -> None:
+        self.changed = False
+        self._materialized: Dict[str, str] = {}  # env vars we already loaded
+        self._raw: Dict[str, str] = {}  # could still need replacement
+        self._needs_replacement: List[str] = []  # env vars that need replacement
+        self._env_files: List[str] = []
+        self._replacer: Replacer = lambda s, c: s
+        self._name, self._env_name, self._root = name, env_name, root
         from .loader.ini.replace import find_replace_part
 
         for line in raw.splitlines():
             if line.strip():
-                try:
-                    key, value = self._extract_key_value(line)
-                    if "{" in key:
-                        raise ValueError(f"invalid line {line!r} in set_env")
-                except ValueError:
-                    _, __, match = find_replace_part(line, 0)
-                    if match:
-                        self._later.append(line)
-                    else:
-                        raise
+                if line.startswith("file|"):
+                    self._env_files.append(line[len("file|") :])
                 else:
-                    self._raw[key] = value
-        self._materialized: Dict[str, str] = {}
-        self.changed = False
+                    try:
+                        key, value = self._extract_key_value(line)
+                        if "{" in key:
+                            raise ValueError(f"invalid line {line!r} in set_env")
+                    except ValueError:
+                        _, __, match = find_replace_part(line, 0)
+                        if match:
+                            self._needs_replacement.append(line)
+                        else:
+                            raise
+                    else:
+                        self._raw[key] = value
+
+    def use_replacer(self, value: Replacer, args: ConfigLoadArgs) -> None:
+        self._replacer = value
+        for filename in self._env_files:
+            self._read_env_file(filename, args)
+
+    def _read_env_file(self, filename: str, args: ConfigLoadArgs) -> None:
+        # Our rules in the documentation, some upstream environment file rules (we follow mostly the docker one):
+        # - https://www.npmjs.com/package/dotenv#rules
+        # - https://docs.docker.com/compose/env-file/
+        env_file = Path(self._replacer(filename, args.copy()))  # apply any replace options
+        env_file = env_file if env_file.is_absolute() else self._root / env_file
+        if not env_file.exists():
+            raise Fail(f"{env_file} does not exist for set_env")
+        for env_line in env_file.read_text().splitlines():
+            env_line = env_line.strip()
+            if not env_line or env_line.startswith("#"):
+                continue
+            key, value = self._extract_key_value(env_line)
+            self._raw[key] = value
 
     @staticmethod
     def _extract_key_value(line: str) -> Tuple[str, str]:
@@ -39,12 +65,12 @@ class SetEnv:
             raise ValueError(f"invalid line {line!r} in set_env")
 
     def load(self, item: str, args: Optional[ConfigLoadArgs] = None) -> str:
-        args = ConfigLoadArgs([], self._name, self._env_name) if args is None else args
-        args.chain.append(f"env:{item}")
         if item in self._materialized:
             return self._materialized[item]
         raw = self._raw[item]
-        result = self.replacer(raw, args)  # apply any replace options
+        args = ConfigLoadArgs([], self._name, self._env_name) if args is None else args
+        args.chain.append(f"env:{item}")
+        result = self._replacer(raw, args)  # apply any replace options
         result = result.replace(r"\#", "#")  # unroll escaped comment with replacement
         self._materialized[item] = result
         self._raw.pop(item, None)  # if the replace requires the env we may be called again, so allow pop to fail
@@ -57,9 +83,9 @@ class SetEnv:
         # start with the materialized ones, maybe we don't need to materialize the raw ones
         yield from self._materialized.keys()
         yield from list(self._raw.keys())  # iterating over this may trigger materialization and change the dict
-        while self._later:
-            line = self._later.pop(0)
-            expanded_line = self.replacer(line, ConfigLoadArgs([], self._name, self._env_name))
+        while self._needs_replacement:
+            line = self._needs_replacement.pop(0)
+            expanded_line = self._replacer(line, ConfigLoadArgs([], self._name, self._env_name))
             sub_raw = dict(self._extract_key_value(sub_line) for sub_line in expanded_line.splitlines() if sub_line)
             self._raw.update(sub_raw)
             yield from sub_raw.keys()
diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py
index d4aeedc5..bed77020 100644
--- a/src/tox/config/sets.py
+++ b/src/tox/config/sets.py
@@ -231,13 +231,14 @@ class EnvConfigSet(ConfigSet):
         def set_env_factory(raw: object) -> SetEnv:
             if not isinstance(raw, str):
                 raise TypeError(raw)
-            return SetEnv(raw, self.name, self.env_name)
+            return SetEnv(raw, self.name, self.env_name, root)
 
+        root = self._conf.core["tox_root"]
         self.add_config(
             keys=["set_env", "setenv"],
             of_type=SetEnv,
             factory=set_env_factory,
-            default=SetEnv("", self.name, self.env_name),
+            default=SetEnv("", self.name, self.env_name, root),
             desc="environment variables to set when running commands in the tox environment",
             post_process=set_env_post_process,
         )
diff --git a/src/tox/pytest.py b/src/tox/pytest.py
index e16677a1..a075d02d 100644
--- a/src/tox/pytest.py
+++ b/src/tox/pytest.py
@@ -166,6 +166,8 @@ class ToxProject:
                 ToxProject._setup_files(at_path, None, value)
             elif isinstance(value, str):
                 at_path.write_text(textwrap.dedent(value))
+            elif value is None:
+                at_path.mkdir()
             else:
                 msg = f"could not handle {at_path / key} with content {value!r}"  # pragma: no cover
                 raise TypeError(msg)  # pragma: no cover
diff --git a/src/tox/tox_env/errors.py b/src/tox/tox_env/errors.py
index fd3d6217..49194b44 100644
--- a/src/tox/tox_env/errors.py
+++ b/src/tox/tox_env/errors.py
@@ -1,13 +1,13 @@
 """Defines tox error types"""
 
 
-class Recreate(RuntimeError):
+class Recreate(Exception):  # noqa: N818
     """Recreate the tox environment"""
 
 
-class Skip(RuntimeError):
+class Skip(Exception):  # noqa: N818
     """Skip this tox environment"""
 
 
-class Fail(RuntimeError):
+class Fail(Exception):  # noqa: N818
     """Failed creating env"""
diff --git a/tests/config/test_set_env.py b/tests/config/test_set_env.py
index 0694a71a..001877d4 100644
--- a/tests/config/test_set_env.py
+++ b/tests/config/test_set_env.py
@@ -1,4 +1,6 @@
-from typing import Callable
+import sys
+from pathlib import Path
+from typing import Any, Dict, Optional
 
 import pytest
 from pytest_mock import MockerFixture
@@ -6,9 +8,14 @@ from pytest_mock import MockerFixture
 from tox.config.set_env import SetEnv
 from tox.pytest import MonkeyPatch, ToxProjectCreator
 
+if sys.version_info >= (3, 8):  # pragma: no cover (py38+)
+    from typing import Protocol
+else:  # pragma: no cover (<py38)
+    from typing_extensions import Protocol
+
 
 def test_set_env_explicit() -> None:
-    set_env = SetEnv("\nA=1\nB = 2\nC= 3\nD= 4", "py", "py")
+    set_env = SetEnv("\nA=1\nB = 2\nC= 3\nD= 4", "py", "py", Path())
     set_env.update({"E": "5 ", "F": "6"}, override=False)
 
     keys = list(set_env)
@@ -23,17 +30,21 @@ def test_set_env_explicit() -> None:
 
 def test_set_env_bad_line() -> None:
     with pytest.raises(ValueError, match="A"):
-        SetEnv("A", "py", "py")
+        SetEnv("A", "py", "py", Path())
 
 
-EvalSetEnv = Callable[[str], SetEnv]
+class EvalSetEnv(Protocol):
+    def __call__(
+        self, tox_ini: str, extra_files: Optional[Dict[str, Any]] = ..., from_cwd: Optional[Path] = ...  # noqa: U100
+    ) -> SetEnv:
+        ...
 
 
 @pytest.fixture()
 def eval_set_env(tox_project: ToxProjectCreator) -> EvalSetEnv:
-    def func(tox_ini: str) -> SetEnv:
-        prj = tox_project({"tox.ini": tox_ini})
-        result = prj.run("c", "-k", "set_env", "-e", "py")
+    def func(tox_ini: str, extra_files: Optional[Dict[str, Any]] = None, from_cwd: Optional[Path] = None) -> SetEnv:
+        prj = tox_project({"tox.ini": tox_ini, **(extra_files or {})})
+        result = prj.run("c", "-k", "set_env", "-e", "py", from_cwd=None if from_cwd is None else prj.path / from_cwd)
         result.assert_success()
         set_env: SetEnv = result.env_conf("py")["set_env"]
         return set_env
@@ -110,3 +121,33 @@ def test_set_env_replacer(eval_set_env: EvalSetEnv, monkeypatch: MonkeyPatch) ->
 def test_set_env_honor_override(eval_set_env: EvalSetEnv) -> None:
     set_env = eval_set_env("[testenv]\npackage=skip\nset_env=PIP_DISABLE_PIP_VERSION_CHECK=0")
     assert set_env.load("PIP_DISABLE_PIP_VERSION_CHECK") == "0"
+
+
+def test_set_env_environment_file(eval_set_env: EvalSetEnv) -> None:
+    env_file = """
+    A=1
+    B= 2
+    C = 1
+    # D = comment # noqa: E800
+    E = "1"
+    F =
+    """
+    extra = {"A": {"a.txt": env_file}, "B": None, "C": None}
+    ini = "[testenv]\npackage=skip\nset_env=file|A{/}a.txt\nchange_dir=C"
+    set_env = eval_set_env(ini, extra_files=extra, from_cwd=Path("B"))
+    content = {k: set_env.load(k) for k in set_env}
+    assert content == {
+        "PIP_DISABLE_PIP_VERSION_CHECK": "1",
+        "A": "1",
+        "B": "2",
+        "C": "1",
+        "E": '"1"',
+        "F": "",
+    }
+
+
+def test_set_env_environment_file_missing(tox_project: ToxProjectCreator) -> None:
+    project = tox_project({"tox.ini": "[testenv]\npackage=skip\nset_env=file|magic.txt"})
+    result = project.run("r")
+    result.assert_failed()
+    assert f"py: failed with {project.path / 'magic.txt'} does not exist for set_env" in result.out