Rewrite substitution parser (#2861)

3 files changed, 192 insertions, 18 deletions

diff --git a/tests/config/loader/ini/replace/test_replace.py b/tests/config/loader/ini/replace/test_replace.py
index 39aab9d3..54bf2a7e 100644
--- a/tests/config/loader/ini/replace/test_replace.py
+++ b/tests/config/loader/ini/replace/test_replace.py
@@ -2,26 +2,92 @@ from __future__ import annotations
 
 import pytest
 
-from tox.config.loader.ini.replace import find_replace_part
+from tests.config.loader.ini.replace.conftest import ReplaceOne
+from tox.config.loader.ini.replace import MatchExpression, find_replace_expr
+from tox.report import HandledError
 
 
 @pytest.mark.parametrize(
-    ("value", "result"),
+    ("value", "exp_output"),
     [
-        ("[]", (0, 1, "posargs")),
-        ("123[]", (3, 4, "posargs")),
-        ("[]123", (0, 1, "posargs")),
-        (r"\[\] []", (5, 6, "posargs")),
-        (r"[\] []", (4, 5, "posargs")),
-        (r"\[] []", (4, 5, "posargs")),
-        ("{foo}", (0, 4, "foo")),
-        (r"\{foo} {bar}", (7, 11, "bar")),
-        ("{foo} {bar}", (0, 4, "foo")),
-        (r"{foo\} {bar}", (7, 11, "bar")),
-        (r"{foo:{bar}}", (5, 9, "bar")),
-        (r"{\{}", (0, 3, r"\{")),
-        (r"{\}}", (0, 3, r"\}")),
+        ("[]", [MatchExpression([["posargs"]])]),
+        ("123[]", ["123", MatchExpression([["posargs"]])]),
+        ("[]123", [MatchExpression([["posargs"]]), "123"]),
+        (r"\[\] []", ["[] ", MatchExpression([["posargs"]])]),
+        (r"[\] []", ["[] ", MatchExpression([["posargs"]])]),
+        (r"\[] []", ["[] ", MatchExpression([["posargs"]])]),
+        ("{foo}", [MatchExpression([["foo"]])]),
+        (r"\{foo} {bar}", ["{foo} ", MatchExpression([["bar"]])]),
+        ("{foo} {bar}", [MatchExpression([["foo"]]), " ", MatchExpression([["bar"]])]),
+        (r"{foo\} {bar}", ["{foo} ", MatchExpression([["bar"]])]),
+        (r"{foo:{bar}}", [MatchExpression([["foo"], [MatchExpression([["bar"]])]])]),
+        (r"{foo\::{bar}}", [MatchExpression([["foo:"], [MatchExpression([["bar"]])]])]),
+        (r"{foo:B:c:D:e}", [MatchExpression([["foo"], ["B"], ["c"], ["D"], ["e"]])]),
+        (r"{\{}", [MatchExpression([["{"]])]),
+        (r"{\}}", [MatchExpression([["}"]])]),
+        (
+            r"p{foo:b{a{r}:t}:{ba}z}s",
+            [
+                "p",
+                MatchExpression(
+                    [
+                        ["foo"],
+                        [
+                            "b",
+                            MatchExpression(
+                                [
+                                    ["a", MatchExpression([["r"]])],
+                                    ["t"],
+                                ],
+                            ),
+                        ],
+                        [
+                            MatchExpression(
+                                [["ba"]],
+                            ),
+                            "z",
+                        ],
+                    ],
+                ),
+                "s",
+            ],
+        ),
+        ("\\", ["\\"]),
+        (r"\d", ["\\d"]),
+        (r"C:\WINDOWS\foo\bar", [r"C:\WINDOWS\foo\bar"]),
     ],
 )
-def test_match(value: str, result: tuple[int, int, str]) -> None:
-    assert find_replace_part(value, 0) == result
+def test_match_expr(value: str, exp_output: list[str | MatchExpression]) -> None:
+    assert find_replace_expr(value) == exp_output
+
+
+@pytest.mark.parametrize(
+    ("value", "exp_exception"),
+    [
+        ("py-{foo,bar}", None),
+        ("py37-{base,i18n},b", None),
+        ("py37-{i18n,base},b", None),
+        ("{toxinidir,}", None),
+        ("{env}", r"MatchError\('No variable name was supplied in {env} substitution'\)"),
+    ],
+)
+def test_dont_replace(replace_one: ReplaceOne, value: str, exp_exception: str | None) -> None:
+    """Test that invalid expressions are not replaced."""
+    if exp_exception:
+        with pytest.raises(HandledError, match=exp_exception):
+            replace_one(value)
+    else:
+        assert replace_one(value) == value
+
+
+@pytest.mark.parametrize(
+    ("match_expression", "exp_repr"),
+    [
+        (MatchExpression([["posargs"]]), "MatchExpression(expr=[['posargs']], term_pos=None)"),
+        (MatchExpression([["posargs"]], 1), "MatchExpression(expr=[['posargs']], term_pos=1)"),
+        (MatchExpression("foo", -42), "MatchExpression(expr='foo', term_pos=-42)"),
+    ],
+)
+def test_match_expression_repr(match_expression: MatchExpression, exp_repr: str) -> None:
+    print(match_expression)
+    assert repr(match_expression) == exp_repr
diff --git a/tests/config/loader/ini/replace/test_replace_env_var.py b/tests/config/loader/ini/replace/test_replace_env_var.py
index 9164a1e7..c9f26400 100644
--- a/tests/config/loader/ini/replace/test_replace_env_var.py
+++ b/tests/config/loader/ini/replace/test_replace_env_var.py
@@ -1,5 +1,10 @@
 from __future__ import annotations
 
+import threading
+from typing import Generator
+
+import pytest
+
 from tests.config.loader.ini.replace.conftest import ReplaceOne
 from tox.pytest import MonkeyPatch
 
@@ -11,6 +16,43 @@ def test_replace_env_set(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> N
     assert result == "something good"
 
 
+def test_replace_env_set_double_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Double backslash should escape to single backslash and not affect surrounding replacements."""
+    monkeypatch.setenv("MAGIC", "something good")
+    result = replace_one(r"{env:MAGIC}\\{env:MAGIC}")
+    assert result == r"something good\something good"
+
+
+def test_replace_env_set_triple_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Triple backslash should escape to single backslash also escape subsequent replacement."""
+    monkeypatch.setenv("MAGIC", "something good")
+    result = replace_one(r"{env:MAGIC}\\\{env:MAGIC}")
+    assert result == r"something good\{env:MAGIC}"
+
+
+def test_replace_env_set_quad_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Quad backslash should escape to two backslashes and not affect surrounding replacements."""
+    monkeypatch.setenv("MAGIC", "something good")
+    result = replace_one(r"\\{env:MAGIC}\\\\{env:MAGIC}\\")
+    assert result == r"\something good\\something good" + "\\"
+
+
+def test_replace_env_when_value_is_backslash(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """When the replacement value is backslash, it shouldn't affect the next replacement."""
+    monkeypatch.setenv("MAGIC", "tragic")
+    monkeypatch.setenv("BS", "\\")
+    result = replace_one(r"{env:BS}{env:MAGIC}")
+    assert result == r"\tragic"
+
+
+def test_replace_env_when_value_is_stuff_then_backslash(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """When the replacement value is a string containing backslash, it shouldn't affect the next replacement."""
+    monkeypatch.setenv("MAGIC", "tragic")
+    monkeypatch.setenv("BS", "stuff\\")
+    result = replace_one(r"{env:BS}{env:MAGIC}")
+    assert result == r"stuff\tragic"
+
+
 def test_replace_env_missing(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
     """If we have a factor that is not specified within the core env-list then that's also an environment"""
     monkeypatch.delenv("MAGIC", raising=False)
@@ -34,14 +76,60 @@ def test_replace_env_missing_default_from_env(replace_one: ReplaceOne, monkeypat
 
 
 def test_replace_env_var_circular(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
-    """If we have a factor that is not specified within the core env-list then that's also an environment"""
+    """Replacement values will not infinitely loop"""
     monkeypatch.setenv("MAGIC", "{env:MAGIC}")
     result = replace_one("{env:MAGIC}")
     assert result == "{env:MAGIC}"
 
 
+@pytest.fixture()
+def reset_env_var_after_delay(monkeypatch: MonkeyPatch) -> Generator[threading.Thread, None, None]:
+    timeout = 2
+
+    def avoid_infinite_loop() -> None:  # pragma: no cover
+        monkeypatch.setenv("TRAGIC", f"envvar forcibly reset after {timeout} sec")
+
+    timer = threading.Timer(2, avoid_infinite_loop)
+    timer.start()
+    yield timer
+    timer.cancel()
+    timer.join()
+
+
+@pytest.mark.usefixtures("reset_env_var_after_delay")
+def test_replace_env_var_circular_flip_flop(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Replacement values will not infinitely loop back and forth"""
+    monkeypatch.setenv("TRAGIC", "{env:MAGIC}")
+    monkeypatch.setenv("MAGIC", "{env:TRAGIC}")
+    result = replace_one("{env:MAGIC}")
+    assert result == "{env:TRAGIC}"
+
+
+@pytest.mark.parametrize("fallback", [True, False])
+def test_replace_env_var_chase(replace_one: ReplaceOne, monkeypatch: MonkeyPatch, fallback: bool) -> None:
+    """Resolve variable to be replaced and default value via indirection."""
+    monkeypatch.setenv("WALK", "THIS")
+    def_val = "or that one"
+    monkeypatch.setenv("DEF", def_val)
+    if fallback:
+        monkeypatch.delenv("THIS", raising=False)
+        exp_result = def_val
+    else:
+        this_val = "path"
+        monkeypatch.setenv("THIS", this_val)
+        exp_result = this_val
+    result = replace_one("{env:{env:WALK}:{env:DEF}}")
+    assert result == exp_result
+
+
 def test_replace_env_default_with_colon(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
     """If we have a factor that is not specified within the core env-list then that's also an environment"""
     monkeypatch.delenv("MAGIC", raising=False)
     result = replace_one("{env:MAGIC:https://some.url.org}")
     assert result == "https://some.url.org"
+
+
+def test_replace_env_default_deep(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Get the value through a long tree of nested defaults."""
+    monkeypatch.delenv("M", raising=False)
+    assert replace_one("{env:M:{env:M:{env:M:{env:M:{env:M:foo}}}}}") == "foo"
diff --git a/tests/config/loader/ini/replace/test_replace_os_sep.py b/tests/config/loader/ini/replace/test_replace_os_sep.py
index 04920c7b..2ab0e467 100644
--- a/tests/config/loader/ini/replace/test_replace_os_sep.py
+++ b/tests/config/loader/ini/replace/test_replace_os_sep.py
@@ -2,9 +2,29 @@ from __future__ import annotations
 
 import os
 
+import pytest
+
 from tests.config.loader.ini.replace.conftest import ReplaceOne
+from tox.pytest import MonkeyPatch
 
 
 def test_replace_os_sep(replace_one: ReplaceOne) -> None:
     result = replace_one("{/}")
     assert result == os.sep
+
+
+@pytest.mark.parametrize("sep", ["/", "\\"])
+def test_replace_os_sep_before_curly(monkeypatch: MonkeyPatch, replace_one: ReplaceOne, sep: str) -> None:
+    """Explicit test case for issue #2732 (windows only)."""
+    monkeypatch.setattr(os, "sep", sep)
+    monkeypatch.delenv("_", raising=False)
+    result = replace_one("{/}{env:_:foo}")
+    assert result == os.sep + "foo"
+
+
+@pytest.mark.parametrize("sep", ["/", "\\"])
+def test_replace_os_sep_sub_exp_regression(monkeypatch: MonkeyPatch, replace_one: ReplaceOne, sep: str) -> None:
+    monkeypatch.setattr(os, "sep", sep)
+    monkeypatch.delenv("_", raising=False)
+    result = replace_one("{env:_:{posargs}{/}.{posargs}}", ["foo"])
+    assert result == f"foo{os.sep}.foo"