Better handling of set_env (#1784)

24 files changed, 298 insertions, 99 deletions

diff --git a/src/tox/config/cli/ini.py b/src/tox/config/cli/ini.py
index 9c98e470..c7bdae34 100644
--- a/src/tox/config/cli/ini.py
+++ b/src/tox/config/cli/ini.py
@@ -51,7 +51,7 @@ class IniConfig:
                     result = None
                 else:
                     source = "file"
-                    value = self.ini.load(key, of_type=of_type, conf=None, env_name="tox")
+                    value = self.ini.load(key, of_type=of_type, conf=None, env_name="tox", chain=[key])
                     result = value, source
             except KeyError:  # just not found
                 result = None
diff --git a/src/tox/config/loader/api.py b/src/tox/config/loader/api.py
index b1eb433f..3a542653 100644
--- a/src/tox/config/loader/api.py
+++ b/src/tox/config/loader/api.py
@@ -1,6 +1,8 @@
 from abc import abstractmethod
 from argparse import ArgumentTypeError
-from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Set, Type, TypeVar
+from concurrent.futures import Future
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Any, Generator, List, Mapping, Optional, Set, Type, TypeVar
 
 from tox.plugin.impl import impl
 
@@ -69,7 +71,9 @@ class Loader(Convert[T]):
     def __repr__(self) -> str:
         return f"{type(self).__name__}"
 
-    def load(self, key: str, of_type: Type[V], conf: Optional["Config"], env_name: Optional[str]) -> V:
+    def load(
+        self, key: str, of_type: Type[V], conf: Optional["Config"], env_name: Optional[str], chain: List[str]
+    ) -> V:
         """
         Load a value.
 
@@ -82,9 +86,25 @@ class Loader(Convert[T]):
         if key in self.overrides:
             return _STR_CONVERT.to(self.overrides[key].value, of_type)
         raw = self.load_raw(key, conf, env_name)
-        converted = self.to(raw, of_type)
+        future: "Future[V]" = Future()
+        with self.build(future, key, of_type, conf, env_name, raw, chain) as prepared:
+            converted = self.to(prepared, of_type)
+            future.set_result(converted)
         return converted
 
+    @contextmanager
+    def build(
+        self,
+        future: "Future[V]",
+        key: str,
+        of_type: Type[V],
+        conf: Optional["Config"],
+        env_name: Optional[str],
+        raw: T,
+        chain: List[str],
+    ) -> Generator[T, None, None]:
+        yield raw
+
 
 @impl
 def tox_add_option(parser: "ToxParser") -> None:
diff --git a/src/tox/config/loader/ini/__init__.py b/src/tox/config/loader/ini/__init__.py
index 0b013557..46042fd6 100644
--- a/src/tox/config/loader/ini/__init__.py
+++ b/src/tox/config/loader/ini/__init__.py
@@ -1,11 +1,15 @@
+import inspect
+from concurrent.futures import Future
 from configparser import ConfigParser, SectionProxy
-from typing import List, Optional, Set, TypeVar
+from contextlib import contextmanager
+from typing import Generator, List, Optional, Set, Type, TypeVar
 
 from tox.config.loader.api import Loader, Override
 from tox.config.loader.ini.factor import filter_for_env
 from tox.config.loader.ini.replace import replace
 from tox.config.loader.str_convert import StrConvert
 from tox.config.main import Config
+from tox.config.set_env import SetEnv
 from tox.report import HandledError
 
 V = TypeVar("V")
@@ -28,15 +32,42 @@ class IniLoader(StrConvert, Loader[str]):
         value = self._section[key]
         collapsed_newlines = value.replace("\\\r\n", "").replace("\\\n", "")  # collapse explicit new-line escape
         if conf is None:  # conf is None when we're loading the global tox configuration file for the CLI
-            replaced = collapsed_newlines  # we don't support factor and replace functionality there
+            factor_filtered = collapsed_newlines  # we don't support factor and replace functionality there
         else:
-            factor_selected = filter_for_env(collapsed_newlines, env_name)  # select matching factors
-            try:
-                replaced = replace(factor_selected, conf, env_name, self)  # do replacements
-            except Exception as exception:
-                msg = f"replace failed in {'tox' if env_name is None else env_name}.{key} with {exception!r}"
-                raise HandledError(msg)
-        return replaced
+            factor_filtered = filter_for_env(collapsed_newlines, env_name)  # select matching factors
+        return factor_filtered
+
+    @contextmanager
+    def build(
+        self,
+        future: "Future[V]",
+        key: str,
+        of_type: Type[V],
+        conf: Optional["Config"],
+        env_name: Optional[str],
+        raw: str,
+        chain: List[str],
+    ) -> Generator[str, None, None]:
+        delay_replace = inspect.isclass(of_type) and issubclass(of_type, SetEnv)
+
+        def replacer(raw_: str, chain_: List[str]) -> str:
+            if conf is None:
+                replaced = raw_  # no replacement supported in the core section
+            else:
+                try:
+                    replaced = replace(conf, env_name, self, raw_, chain_)  # do replacements
+                except Exception as exception:
+                    msg = f"replace failed in {'tox' if env_name is None else env_name}.{key} with {exception!r}"
+                    raise HandledError(msg) from exception
+            return replaced
+
+        if not delay_replace:
+            raw = replacer(raw, chain)
+        yield raw
+        if delay_replace:
+            converted = future.result()
+            if hasattr(converted, "replacer"):  # pragma: no branch
+                converted.replacer = replacer  # type: ignore[attr-defined]
 
     def found_keys(self) -> Set[str]:
         return set(self._section.keys())
diff --git a/src/tox/config/loader/ini/factor.py b/src/tox/config/loader/ini/factor.py
index 017749ff..bd1b5e63 100644
--- a/src/tox/config/loader/ini/factor.py
+++ b/src/tox/config/loader/ini/factor.py
@@ -15,12 +15,13 @@ def filter_for_env(value: str, name: Optional[str]) -> str:
     overall = []
     for factors, content in expand_factors(value):
         if factors is None:
-            overall.append(content)
+            if content:
+                overall.append(content)
         else:
             for group in factors:
                 for a_name, negate in group:
                     contains = a_name in current
-                    if contains == negate:
+                    if not ((contains is True and negate is False) or (contains is False and negate is True)):
                         break
                 else:
                     overall.append(content)
diff --git a/src/tox/config/loader/ini/replace.py b/src/tox/config/loader/ini/replace.py
index 3a6bb98f..f590b3dd 100644
--- a/src/tox/config/loader/ini/replace.py
+++ b/src/tox/config/loader/ini/replace.py
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Iterator, List, Optional, Sequence, Tuple, Uni
 
 from tox.config.loader.stringify import stringify
 from tox.config.main import Config
+from tox.config.set_env import SetEnv
 from tox.config.sets import ConfigSet
 from tox.execute.request import shell_cmd
 
@@ -18,18 +19,19 @@ if TYPE_CHECKING:
 CORE_PREFIX = "tox"
 BASE_TEST_ENV = "testenv"
 
-ARGS_GROUP = re.compile(r"(?<!\\):")
+# split alongside :, unless it's esscaped, or it's preceded by a single capital letter (Windows drive letter in paths)
+ARGS_GROUP = re.compile(r"(?<!\\\\|:[A-Z]):")
 
 
-def replace(value: str, conf: Config, name: Optional[str], loader: "IniLoader") -> str:
+def replace(conf: Config, name: Optional[str], loader: "IniLoader", value: str, chain: List[str]) -> str:
     # perform all non-escaped replaces
     start, end = 0, 0
     while True:
-        start, end, match = _find_replace_part(value, start, end)
+        start, end, match = find_replace_part(value, start, end)
         if not match:
             break
         to_replace = value[start + 1 : end]
-        replaced = _replace_match(conf, name, loader, to_replace)
+        replaced = _replace_match(conf, name, loader, to_replace, chain.copy())
         if replaced is None:
             # if we cannot replace, keep what was there, and continue looking for additional replaces following
             # note, here we cannot raise because the content may be a factorial expression, and in those case we don't
@@ -47,7 +49,7 @@ def replace(value: str, conf: Config, name: Optional[str], loader: "IniLoader")
     return value
 
 
-def _find_replace_part(value: str, start: int, end: int) -> Tuple[int, int, bool]:
+def find_replace_part(value: str, start: int, end: int) -> Tuple[int, int, bool]:
     match = False
     while end != -1:
         end = value.find("}", end)
@@ -68,16 +70,20 @@ def _find_replace_part(value: str, start: int, end: int) -> Tuple[int, int, bool
     return start, end, match
 
 
-def _replace_match(conf: Config, current_env: Optional[str], loader: "IniLoader", value: str) -> Optional[str]:
+def _replace_match(
+    conf: Config, current_env: Optional[str], loader: "IniLoader", value: str, chain: List[str]
+) -> Optional[str]:
     of_type, *args = ARGS_GROUP.split(value)
-    if of_type == "env":
-        replace_value: Optional[str] = replace_env(args)
+    if of_type == "/":
+        replace_value: Optional[str] = os.sep
+    elif of_type == "env":
+        replace_value = replace_env(conf, current_env, args, chain)
     elif of_type == "tty":
         replace_value = replace_tty(args)
     elif of_type == "posargs":
         replace_value = replace_pos_args(args, conf.pos_args)
     else:
-        replace_value = replace_reference(conf, current_env, loader, value)
+        replace_value = replace_reference(conf, current_env, loader, value, chain)
     return replace_value
 
 
@@ -96,6 +102,7 @@ def replace_reference(
     current_env: Optional[str],
     loader: "IniLoader",
     value: str,
+    chain: List[str],
 ) -> Optional[str]:
     # a return value of None indicates could not replace
     match = _REPLACE_REF.match(value)
@@ -112,7 +119,7 @@ def replace_reference(
                 try:
                     if isinstance(src, SectionProxy):
                         return src[key]
-                    value = src[key]
+                    value = src.load(key, chain)
                     as_str, _ = stringify(value)
                     return as_str
                 except KeyError as exc:  # if fails, keep trying maybe another source can satisfy
@@ -172,10 +179,24 @@ def replace_pos_args(args: List[str], pos_args: Optional[Sequence[str]]) -> str:
     return replace_value
 
 
-def replace_env(args: List[str]) -> str:
+def replace_env(conf: Config, env_name: Optional[str], args: List[str], chain: List[str]) -> str:
     key = args[0]
-    default = "" if len(args) == 1 else args[1]
-    return os.environ.get(key, default)
+    new_key = f"env:{key}"
+
+    if env_name is not None:  # on core no set env support # pragma: no branch
+        if new_key not in chain:  # check if set env
+            chain.append(new_key)
+            env_conf = conf.get_env(env_name)
+            set_env: SetEnv = env_conf["set_env"]
+            if key in set_env:
+                return set_env.load(key, chain)
+        elif chain[-1] != new_key:  # if there's a chain but only self-refers than use os.environ
+            raise ValueError(f"circular chain between set env {', '.join(i[4:] for i in chain[chain.index(new_key):])}")
+
+    if key in os.environ:
+        return os.environ[key]
+
+    return "" if len(args) == 1 else args[1]
 
 
 def replace_tty(args: List[str]) -> str:
@@ -190,4 +211,5 @@ __all__ = (
     "CORE_PREFIX",
     "BASE_TEST_ENV",
     "replace",
+    "find_replace_part",
 )
diff --git a/src/tox/config/loader/str_convert.py b/src/tox/config/loader/str_convert.py
index f5c5acd0..7635b5c5 100644
--- a/src/tox/config/loader/str_convert.py
+++ b/src/tox/config/loader/str_convert.py
@@ -1,10 +1,9 @@
 """Convert string configuration values to tox python configuration objects."""
-import re
 import shlex
 import sys
 from itertools import chain
 from pathlib import Path
-from typing import Any, Iterator, Tuple, Type
+from typing import Any, Iterator, List, Tuple, Type
 
 from tox.config.loader.convert import Convert
 from tox.config.types import Command, EnvList
@@ -51,18 +50,21 @@ class StrConvert(Convert[str]):
 
     @staticmethod
     def to_command(value: str) -> Command:
-        win = sys.platform == "win32"
-        splitter = shlex.shlex(value, posix=not win)
+        is_win = sys.platform == "win32"
+        splitter = shlex.shlex(value, posix=not is_win)
         splitter.whitespace_split = True
-        if win:  # pragma: win32 cover
-            args = []
+        if is_win:  # pragma: win32 cover
+            args: List[str] = []
             for arg in splitter:
-                if arg[0] == "'" and arg[-1] == "'":  # remove outer quote - the arg is passed as one, so no need for it
+                # on Windows quoted arguments will remain quoted, strip it
+                if (
+                    len(arg) > 1
+                    and (arg.startswith('"') and arg.endswith('"'))
+                    or (arg.startswith("'") and arg.endswith("'"))
+                ):
                     arg = arg[1:-1]
-                if "/" in arg:  # normalize posix paths to nt paths
-                    arg = "\\".join(re.split(pattern=r"[\\/]", string=arg))
                 args.append(arg)
-        else:  # pragma: win32 no cover
+        else:
             args = list(splitter)
         return Command(args)
 
diff --git a/src/tox/config/loader/stringify.py b/src/tox/config/loader/stringify.py
index 85dddac6..da432252 100644
--- a/src/tox/config/loader/stringify.py
+++ b/src/tox/config/loader/stringify.py
@@ -2,6 +2,7 @@ from enum import Enum
 from pathlib import Path
 from typing import Any, Mapping, Sequence, Set, Tuple
 
+from tox.config.set_env import SetEnv
 from tox.config.types import Command, EnvList
 
 
@@ -26,6 +27,8 @@ def stringify(value: Any) -> Tuple[str, bool]:
         return "\n".join(e for e in value.envs), True
     if isinstance(value, Command):
         return value.shell, True
+    if isinstance(value, SetEnv):
+        return stringify({k: value.load(k) for k in sorted(list(value))})
     return str(value), False
 
 
diff --git a/src/tox/config/main.py b/src/tox/config/main.py
index 790c6e4e..150002c9 100644
--- a/src/tox/config/main.py
+++ b/src/tox/config/main.py
@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence
 from tox.config.loader.api import Loader, Override, OverrideMap
 from tox.config.source import Source
 
-from .sets import ConfigSet, CoreConfigSet
+from .sets import CoreConfigSet, EnvConfigSet
 
 
 class Config:
@@ -26,9 +26,9 @@ class Config:
             self._overrides[override.namespace].append(override)
 
         self._src = config_source
-        self._env_to_set: Dict[str, ConfigSet] = OrderedDict()
+        self._env_to_set: Dict[str, EnvConfigSet] = OrderedDict()
         self._core_set: Optional[CoreConfigSet] = None
-        self.register_config_set: Callable[[str, ConfigSet], Any] = lambda n, e: None
+        self.register_config_set: Callable[[str, EnvConfigSet], Any] = lambda n, e: None
 
     @property
     def core(self) -> CoreConfigSet:
@@ -44,11 +44,13 @@ class Config:
         self._core_set = core
         return core
 
-    def get_env(self, item: str, package: bool = False, loaders: Optional[Sequence[Loader[Any]]] = None) -> ConfigSet:
+    def get_env(
+        self, item: str, package: bool = False, loaders: Optional[Sequence[Loader[Any]]] = None
+    ) -> EnvConfigSet:
         try:
             return self._env_to_set[item]
         except KeyError:
-            env = ConfigSet(self, item)
+            env = EnvConfigSet(self, item)
             self._env_to_set[item] = env
             if loaders is not None:
                 env.loaders.extend(loaders)
diff --git a/src/tox/config/of_type.py b/src/tox/config/of_type.py
index 2322dfa4..f82b5cc8 100644
--- a/src/tox/config/of_type.py
+++ b/src/tox/config/of_type.py
@@ -23,7 +23,7 @@ class ConfigDefinition(ABC, Generic[T]):
         self.env_name = env_name
 
     @abstractmethod
-    def __call__(self, conf: "Config", key: Optional[str], loaders: List[Loader[T]]) -> T:
+    def __call__(self, conf: "Config", key: Optional[str], loaders: List[Loader[T]], chain: List[str]) -> T:
         raise NotImplementedError
 
     def __eq__(self, o: Any) -> bool:
@@ -46,7 +46,7 @@ class ConfigConstantDefinition(ConfigDefinition[T]):
         super().__init__(keys, desc, env_name)
         self.value = value
 
-    def __call__(self, conf: "Config", name: Optional[str], loaders: List[Loader[T]]) -> T:
+    def __call__(self, conf: "Config", name: Optional[str], loaders: List[Loader[T]], chain: List[str]) -> T:
         if callable(self.value):
             value = self.value()
         else:
@@ -78,13 +78,13 @@ class ConfigDynamicDefinition(ConfigDefinition[T]):
         self.post_process = post_process
         self._cache: Union[object, T] = _PLACE_HOLDER
 
-    def __call__(self, conf: "Config", name: Optional[str], loaders: List[Loader[T]]) -> T:
+    def __call__(self, conf: "Config", name: Optional[str], loaders: List[Loader[T]], chain: List[str]) -> T:
         if self._cache is _PLACE_HOLDER:
             found = False
             for key in self.keys:
                 for loader in loaders:
                     try:
-                        value = loader.load(key, self.of_type, conf, self.env_name)
+                        value = loader.load(key, self.of_type, conf, self.env_name, chain)
                         found = True
                     except KeyError:
                         continue
diff --git a/src/tox/config/set_env.py b/src/tox/config/set_env.py
new file mode 100644
index 00000000..68b365ad
--- /dev/null
+++ b/src/tox/config/set_env.py
@@ -0,0 +1,71 @@
+from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple
+
+Replacer = Callable[[str, List[str]], str]
+
+
+class SetEnv:
+    def __init__(self, raw: str) -> None:
+        self.replacer: Replacer = lambda s, c: s
+        lines = raw.splitlines()
+        self._later: List[str] = []
+        self._raw: Dict[str, str] = {}
+        from .loader.ini.replace import find_replace_part
+
+        for line in lines:
+            if line.strip():
+                try:
+                    key, value = self._extract_key_value(line)
+                    if "{" in key:
+                        raise ValueError(f"invalid line {line!r} in set_env")
+                except ValueError:
+                    _, __, match = find_replace_part(line, 0, 0)
+                    if match:
+                        self._later.append(line)
+                    else:
+                        raise
+                else:
+                    self._raw[key] = value
+        self._materialized: Dict[str, str] = {}
+
+    @staticmethod
+    def _extract_key_value(line: str) -> Tuple[str, str]:
+        try:
+            at = line.index("=")
+        except ValueError:
+            raise ValueError(f"invalid line {line!r} in set_env")
+        key, value = line[:at], line[at + 1 :]
+        return key.strip(), value.strip()
+
+    def load(self, item: str, chain: Optional[List[str]] = None) -> str:
+        if chain is None:
+            chain = [f"env:{item}"]
+        if item in self._materialized:
+            return self._materialized[item]
+        raw = self._raw[item]
+        result = self.replacer(raw, chain)  # apply any replace options
+        self._materialized[item] = result
+        del self._raw[item]
+        return result
+
+    def __contains__(self, item: object) -> bool:
+        return isinstance(item, str) and item in self.__iter__()
+
+    def __iter__(self) -> Iterator[str]:
+        # start with the materialized ones, maybe we don't need to materialize the raw ones
+        yield from self._materialized.keys()
+        yield from list(self._raw.keys())  # iterating over this may trigger materialization and change the dict
+        while self._later:
+            line = self._later.pop(0)
+            expanded_line = self.replacer(line, [])
+            sub_raw = {}
+            for sub_line in expanded_line.splitlines():
+                key, value = self._extract_key_value(sub_line)
+                sub_raw[key] = value
+            self._raw.update(sub_raw)
+            yield from sub_raw.keys()
+
+    def update(self, param: Mapping[str, str]) -> None:
+        self._materialized.update(param)
+
+
+__all__ = ("SetEnv",)
diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py
index 95fdb925..2784872c 100644
--- a/src/tox/config/sets.py
+++ b/src/tox/config/sets.py
@@ -7,6 +7,7 @@ from typing import (
     Dict,
     Iterator,
     List,
+    Mapping,
     Optional,
     Sequence,
     Set,
@@ -17,6 +18,7 @@ from typing import (
 )
 
 from .of_type import ConfigConstantDefinition, ConfigDefinition, ConfigDynamicDefinition
+from .set_env import SetEnv
 from .types import EnvList
 
 if TYPE_CHECKING:
@@ -79,8 +81,16 @@ class ConfigSet:
         return definition
 
     def __getitem__(self, item: str) -> Any:
+        return self.load(item)
+
+    def load(self, item: str, chain: Optional[List[str]] = None) -> Any:
         config_definition = self._defined[item]
-        return config_definition(self._conf, item, self.loaders)
+        if chain is None:
+            chain = []
+        if item in chain:
+            raise ValueError(f"circular chain detected {', '.join(chain[chain.index(item):])}")
+        chain.append(item)
+        return config_definition(self._conf, item, self.loaders, chain)
 
     def __repr__(self) -> str:
         values = (v for v in (f"name={self.name!r}" if self.name else "", f"loaders={self.loaders!r}") if v)
@@ -129,3 +139,28 @@ class CoreConfigSet(ConfigSet):
             default=EnvList([]),
             desc="define environments to automatically run",
         )
+
+
+class EnvConfigSet(ConfigSet):
+    def __init__(self, conf: "Config", name: Optional[str]):
+        super().__init__(conf, name=name)
+        self.default_set_env_loader: Callable[[], Mapping[str, str]] = lambda: {}
+
+        def set_env_post_process(values: SetEnv, config: "Config") -> SetEnv:
+            values.update(self.default_set_env_loader())
+            return values
+
+        self.add_config(
+            keys=["set_env", "setenv"],
+            of_type=SetEnv,
+            default=SetEnv(""),
+            desc="environment variables to set when running commands in the tox environment",
+            post_process=set_env_post_process,
+        )
+
+
+__all__ = (
+    "ConfigSet",
+    "CoreConfigSet",
+    "EnvConfigSet",
+)
diff --git a/src/tox/execute/local_sub_process/__init__.py b/src/tox/execute/local_sub_process/__init__.py
index a8af51a8..65f15f12 100644
--- a/src/tox/execute/local_sub_process/__init__.py
+++ b/src/tox/execute/local_sub_process/__init__.py
@@ -190,7 +190,6 @@ class LocalSubProcessExecuteInstance(ExecuteInstance):
             self.request.env["LINES"] = str(lines)
 
         stdout, stderr = self.get_stream_file_no("stdout"), self.get_stream_file_no("stderr")
-
         try:
             self.process = process = Popen(
                 self.cmd,
diff --git a/src/tox/provision.py b/src/tox/provision.py
index 48b00266..9996d248 100644
--- a/src/tox/provision.py
+++ b/src/tox/provision.py
@@ -73,7 +73,7 @@ def provision(state: State) -> Union[int, bool]:
     for package in requires:
         package_name = canonicalize_name(package.name)
         try:
-            dist = distribution(package_name)
+            dist = distribution(package_name)  # type: ignore[no-untyped-call]
             if not package.specifier.contains(dist.version, prereleases=True):
                 missing.append((package, dist.version))
         except PackageNotFoundError:
diff --git a/src/tox/pytest.py b/src/tox/pytest.py
index 84676dd6..a5bdd508 100644
--- a/src/tox/pytest.py
+++ b/src/tox/pytest.py
@@ -32,6 +32,7 @@ from virtualenv.discovery.py_info import PythonInfo
 from virtualenv.info import IS_WIN, fs_supports_symlink
 
 import tox.run
+from tox.config.sets import EnvConfigSet
 from tox.execute.api import Execute, ExecuteInstance, ExecuteStatus, Outcome
 from tox.execute.request import ExecuteRequest, shell_cmd
 from tox.execute.stream import SyncWrite
@@ -299,6 +300,9 @@ class ToxRunOutcome:
             raise RuntimeError("no state")
         return self._state
 
+    def env_conf(self, name: str) -> EnvConfigSet:
+        return self.state.conf.get_env(name)
+
     @property
     def success(self) -> bool:
         return self.code == Outcome.OK
diff --git a/src/tox/session/cmd/show_config.py b/src/tox/session/cmd/show_config.py
index 759505db..8849b470 100644
--- a/src/tox/session/cmd/show_config.py
+++ b/src/tox/session/cmd/show_config.py
@@ -27,17 +27,20 @@ def tox_add_option(parser: ToxParser) -> None:
 
 
 def show_config(state: State) -> int:
+    show_core = state.options.env.all or state.options.show_core
     keys: List[str] = state.options.list_keys_only
     # environments may define core configuration flags, so we must exhaust first the environments to tell the core part
-    for name in state.env_list(everything=False):
+    envs = list(state.env_list(everything=False))
+    for at, name in enumerate(envs):
         tox_env = state.tox_env(name)
         print(f"[testenv:{name}]")
         if not keys:
             print(f"type = {type(tox_env).__name__}")
         print_conf(tox_env.conf, keys)
-        print("")
+        if show_core or at + 1 != len(envs):
+            print("")
     # no print core
-    if state.options.env.all or state.options.show_core:
+    if show_core:
         print("[tox]")
         print_conf(state.conf.core, keys)
     return 0
diff --git a/src/tox/session/state.py b/src/tox/session/state.py
index 05928765..dab94b1c 100644
--- a/src/tox/session/state.py
+++ b/src/tox/session/state.py
@@ -1,7 +1,7 @@
 from typing import TYPE_CHECKING, Dict, Iterator, Optional, Sequence, Set, Tuple, cast
 
 from tox.config.main import Config
-from tox.config.sets import ConfigSet
+from tox.config.sets import EnvConfigSet
 from tox.journal import Journal
 from tox.plugin.impl import impl
 from tox.report import HandledError, ToxHandler
@@ -64,7 +64,7 @@ class State:
             self.conf.get_env(name)  # the lookup here will trigger register_config_set, which will build it
             return self._run_env[name]
 
-    def register_config_set(self, name: str, config_set: ConfigSet) -> None:
+    def register_config_set(self, name: str, config_set: EnvConfigSet) -> None:
         """Ensure the config set with the given name has been registered with configuration values"""
         # during the creation of hte tox environment we automatically register configurations, so to ensure
         # config sets have a set of defined values in it we have to ensure the tox environment is created
@@ -75,7 +75,7 @@ class State:
         # runtime environments are created upon lookup via the tox_env method, call it
         self._build_run_env(config_set)
 
-    def _build_run_env(self, env_conf: ConfigSet) -> None:
+    def _build_run_env(self, env_conf: EnvConfigSet) -> None:
         env_conf.add_config(
             keys="runner",
             desc="the tox execute used to evaluate this environment",
diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py
index ae678557..1ba40a17 100644
--- a/src/tox/tox_env/api.py
+++ b/src/tox/tox_env/api.py
@@ -13,7 +13,8 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Sequence, Tuple, Union, cast
 
 from tox.config.main import Config
-from tox.config.sets import ConfigSet
+from tox.config.set_env import SetEnv
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.execute.api import Execute, ExecuteStatus, Outcome, StdinSource
 from tox.execute.request import ExecuteRequest
 from tox.journal import EnvJournal
@@ -30,11 +31,11 @@ LOGGER = logging.getLogger(__name__)
 
 class ToxEnv(ABC):
     def __init__(
-        self, conf: ConfigSet, core: ConfigSet, options: "Parsed", journal: EnvJournal, log_handler: ToxHandler
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: "Parsed", journal: EnvJournal, log_handler: ToxHandler
     ) -> None:
         self.journal = journal
-        self.conf: ConfigSet = conf
-        self.core: ConfigSet = core
+        self.conf: EnvConfigSet = conf
+        self.core: CoreConfigSet = core
         self.options = options
         self._executor: Optional[Execute] = None
         self.register_config()
@@ -90,19 +91,7 @@ class ToxEnv(ABC):
             default=lambda conf, name: cast(Path, conf.core["work_dir"]) / cast(str, self.conf["env_name"]) / "tmp",
             desc="a folder that is always reset at the start of the run",
         )
-
-        def set_env_post_process(values: Dict[str, str], config: Config) -> Dict[str, str]:
-            env = self.default_set_env()
-            env.update(values)
-            return env
-
-        self.conf.add_config(
-            keys=["set_env", "setenv"],
-            of_type=Dict[str, str],
-            default={},
-            desc="environment variables to set when running commands in the tox environment",
-            post_process=set_env_post_process,
-        )
+        self.conf.default_set_env_loader = self.default_set_env
 
         def pass_env_post_process(values: List[str], config: Config) -> List[str]:
             values.extend(self.default_pass_env())
@@ -199,7 +188,6 @@ class ToxEnv(ABC):
         if self._env_vars is not None:
             return self._env_vars
         result: Dict[str, str] = {}
-
         pass_env: List[str] = self.conf["pass_env"]
         glob_pass_env = [re.compile(e.replace("*", ".*")) for e in pass_env if "*" in e]
         literal_pass_env = [e for e in pass_env if "*" not in e]
@@ -210,8 +198,9 @@ class ToxEnv(ABC):
             for env, value in os.environ.items():
                 if any(g.match(env) is not None for g in glob_pass_env):
                     result[env] = value
-        set_env: Dict[str, str] = self.conf["set_env"]
-        result.update(set_env)
+        set_env: SetEnv = self.conf["set_env"]
+        for key in set_env:
+            result[key] = set_env.load(key)
         result["PATH"] = os.pathsep.join([str(i) for i in self._paths] + os.environ.get("PATH", "").split(os.pathsep))
         self._env_vars = result
         return result
diff --git a/src/tox/tox_env/package.py b/src/tox/tox_env/package.py
index b32b2c40..88861174 100644
--- a/src/tox/tox_env/package.py
+++ b/src/tox/tox_env/package.py
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, List, Set
 
 from packaging.requirements import Requirement
 
-from tox.config.sets import ConfigSet
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.journal import EnvJournal
 from tox.report import ToxHandler
 from tox.util.threading import AtomicCounter
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
 
 class PackageToxEnv(ToxEnv, ABC):
     def __init__(
-        self, conf: ConfigSet, core: ConfigSet, options: "Parsed", journal: EnvJournal, log_handler: ToxHandler
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: "Parsed", journal: EnvJournal, log_handler: ToxHandler
     ) -> None:
         super().__init__(conf, core, options, journal, log_handler)
         self.recreate_package = options.no_recreate_pkg is False if options.recreate else False
diff --git a/src/tox/tox_env/python/api.py b/src/tox/tox_env/python/api.py
index 28fb5d79..ed7dd27d 100644
--- a/src/tox/tox_env/python/api.py
+++ b/src/tox/tox_env/python/api.py
@@ -12,7 +12,7 @@ from virtualenv.discovery.py_spec import PythonSpec
 
 from tox.config.cli.parser import Parsed
 from tox.config.main import Config
-from tox.config.sets import ConfigSet
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.journal import EnvJournal
 from tox.report import ToxHandler
 from tox.tox_env.api import ToxEnv
@@ -63,7 +63,7 @@ PythonDeps = Sequence[PythonDep]
 
 class Python(ToxEnv, ABC):
     def __init__(
-        self, conf: ConfigSet, core: ConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
     ) -> None:
         self._base_python: Optional[PythonInfo] = None
         self._base_python_searched: bool = False
@@ -83,6 +83,11 @@ class Python(ToxEnv, ABC):
             value=lambda: self.env_site_package_dir(),
         )
         self.conf.add_constant(
+            keys=["env_bin_dir", "envbindir"],
+            desc="the python environments site package",
+            value=lambda: self.env_bin_dir(),
+        )
+        self.conf.add_constant(
             ["env_python", "envpython"],
             desc="python executable from within the tox environment",
             value=lambda: self.env_python(),
@@ -123,6 +128,11 @@ class Python(ToxEnv, ABC):
         """The python executable within the tox environment"""
         raise NotImplementedError
 
+    @abstractmethod
+    def env_bin_dir(self) -> Path:
+        """The binary folder within the tox environment"""
+        raise NotImplementedError
+
     def setup(self) -> None:
         """setup a virtual python environment"""
         conf = self.python_cache()
diff --git a/src/tox/tox_env/python/runner.py b/src/tox/tox_env/python/runner.py
index bdc5d797..f9b0f1af 100644
--- a/src/tox/tox_env/python/runner.py
+++ b/src/tox/tox_env/python/runner.py
@@ -8,7 +8,7 @@ from pathlib import Path
 from typing import Any, Dict, List, Set
 
 from tox.config.cli.parser import Parsed
-from tox.config.sets import ConfigSet
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.journal import EnvJournal
 from tox.report import ToxHandler
 from tox.tox_env.errors import Recreate
@@ -19,7 +19,9 @@ from .req_file import RequirementsFile
 
 
 class PythonRun(Python, RunToxEnv, ABC):
-    def __init__(self, conf: ConfigSet, core: ConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler):
+    def __init__(
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
+    ):
         super().__init__(conf, core, options, journal, log_handler)
         self._packages: List[PythonDep] = []
 
diff --git a/src/tox/tox_env/python/virtual_env/api.py b/src/tox/tox_env/python/virtual_env/api.py
index dd5692c8..5af0a504 100644
--- a/src/tox/tox_env/python/virtual_env/api.py
+++ b/src/tox/tox_env/python/virtual_env/api.py
@@ -10,7 +10,7 @@ from virtualenv.create.creator import Creator
 from virtualenv.run.session import Session
 
 from tox.config.cli.parser import DEFAULT_VERBOSITY, Parsed
-from tox.config.sets import ConfigSet
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.execute.api import Execute, Outcome, StdinSource
 from tox.execute.local_sub_process import LocalSubProcessExecutor
 from tox.journal import EnvJournal
@@ -23,7 +23,7 @@ class VirtualEnv(Python, ABC):
     """A python executor that uses the virtualenv project with pip"""
 
     def __init__(
-        self, conf: ConfigSet, core: ConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
     ) -> None:
         self._virtualenv_session: Optional[Session] = None  # type: ignore[no-any-unimported]
         super().__init__(conf, core, options, journal, log_handler)
@@ -90,6 +90,9 @@ class VirtualEnv(Python, ABC):
     def env_python(self) -> Path:
         return cast(Path, self.creator.exe)
 
+    def env_bin_dir(self) -> Path:
+        return cast(Path, self.creator.script_dir)
+
     def install_python_packages(
         self,
         packages: PythonDeps,
diff --git a/src/tox/tox_env/python/virtual_env/package/api.py b/src/tox/tox_env/python/virtual_env/package/api.py
index 03ea06e8..3df7bb33 100644
--- a/src/tox/tox_env/python/virtual_env/package/api.py
+++ b/src/tox/tox_env/python/virtual_env/package/api.py
@@ -11,7 +11,7 @@ from packaging.markers import Variable
 from packaging.requirements import Requirement
 
 from tox.config.cli.parser import Parsed
-from tox.config.sets import ConfigSet
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.execute.api import ExecuteStatus
 from tox.execute.pep517_backend import LocalSubProcessPep517Executor
 from tox.execute.request import StdinSource
@@ -83,7 +83,7 @@ class Pep517VirtualEnvPackage(VirtualEnv, PythonPackage, Frontend, ABC):
     """local file system python virtual environment via the virtualenv package"""
 
     def __init__(
-        self, conf: ConfigSet, core: ConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: Parsed, journal: EnvJournal, log_handler: ToxHandler
     ) -> None:
         VirtualEnv.__init__(self, conf, core, options, journal, log_handler)
         Frontend.__init__(self, *Frontend.create_args_from_folder(core["tox_root"]))
@@ -122,7 +122,7 @@ class Pep517VirtualEnvPackage(VirtualEnv, PythonPackage, Frontend, ABC):
             return  # pragma: no cover
         self.ensure_setup()
         dist_info = self.prepare_metadata_for_build_wheel(self.meta_folder).metadata
-        self._distribution_meta = Distribution.at(str(dist_info))
+        self._distribution_meta = Distribution.at(str(dist_info))  # type: ignore[no-untyped-call]
 
     @abstractmethod
     def _build_artifact(self) -> Path:
@@ -163,18 +163,16 @@ class Pep517VirtualEnvPackage(VirtualEnv, PythonPackage, Frontend, ABC):
             ):
                 if marker_key.value == "extra" and op.value == "==":  # pragma: no branch
                     extra = marker_value.value
+                    del markers[_at]
+                    _at -= 1
+                    if _at > 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")):
+                        del markers[_at]
+                    if len(markers) == 0:
+                        req.marker = None
                     break
             # continue only if this extra should be included
             if not (extra is None or extra in extras):
                 continue
-            # delete the extra marker if present
-            if _at is not None:
-                del markers[_at]
-                _at -= 1
-                if _at > 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")):
-                    del markers[_at]
-                if len(markers) == 0:
-                    req.marker = None
             result.append(req)
         return result
 
@@ -201,7 +199,9 @@ class Pep517VirtualEnvPackage(VirtualEnv, PythonPackage, Frontend, ABC):
     @property
     def environment_variables(self) -> Dict[str, str]:
         env = super().environment_variables
-        env["PYTHONPATH"] = os.pathsep.join(str(i) for i in self._backend_paths)
+        backend = os.pathsep.join(str(i) for i in self._backend_paths).strip()
+        if backend:
+            env["PYTHONPATH"] = backend
         return env
 
     def teardown(self) -> None:
diff --git a/src/tox/tox_env/runner.py b/src/tox/tox_env/runner.py
index cce84f18..5d5d1872 100644
--- a/src/tox/tox_env/runner.py
+++ b/src/tox/tox_env/runner.py
@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Generator, List, Optional, Tuple, cast
 
-from tox.config.sets import ConfigSet
+from tox.config.sets import CoreConfigSet, EnvConfigSet
 from tox.config.types import Command, EnvList
 from tox.journal import EnvJournal
 from tox.report import ToxHandler
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
 
 class RunToxEnv(ToxEnv, ABC):
     def __init__(
-        self, conf: ConfigSet, core: ConfigSet, options: "Parsed", journal: EnvJournal, log_handler: ToxHandler
+        self, conf: EnvConfigSet, core: CoreConfigSet, options: "Parsed", journal: EnvJournal, log_handler: ToxHandler
     ) -> None:
         self.has_package = False
         self.package_env: Optional[PackageToxEnv] = None
diff --git a/src/tox/util/pep517/via_fresh_subprocess.py b/src/tox/util/pep517/via_fresh_subprocess.py
index 1f83a557..11858e85 100644
--- a/src/tox/util/pep517/via_fresh_subprocess.py
+++ b/src/tox/util/pep517/via_fresh_subprocess.py
@@ -45,7 +45,9 @@ class SubprocessFrontend(Frontend):
         self, cmd: str, result_file: Path, msg: str
     ) -> Iterator[SubprocessCmdStatus]:
         env = os.environ.copy()
-        env["PYTHONPATH"] = os.pathsep.join(str(i) for i in self._backend_paths)
+        backend = os.pathsep.join(str(i) for i in self._backend_paths).strip()
+        if backend:
+            env["PYTHONPATH"] = backend
         process = Popen(
             args=[sys.executable] + self.backend_args,
             stdout=PIPE,