Support extras in (editable) path requirements (#1998)

Signed-off-by: Bernát Gábor <gaborjbernat@gmail.com>

3 files changed, 58 insertions, 21 deletions

diff --git a/docs/changelog/1933.bugfix.rst b/docs/changelog/1933.bugfix.rst
new file mode 100644
index 00000000..320be733
--- /dev/null
+++ b/docs/changelog/1933.bugfix.rst
@@ -0,0 +1 @@
+Support for extras with paths for python deps and requirement files - by :user:`gaborbernat`.
diff --git a/src/tox/tox_env/python/pip/req_file.py b/src/tox/tox_env/python/pip/req_file.py
index 62d05740..cd916e75 100644
--- a/src/tox/tox_env/python/pip/req_file.py
+++ b/src/tox/tox_env/python/pip/req_file.py
@@ -3,7 +3,7 @@ import os
 import re
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union
 
 from packaging.requirements import InvalidRequirement, Requirement
 
@@ -108,25 +108,27 @@ class RequirementWithFlags(Requirement, Flags):
 
 
 class PathReq(PipRequirementEntry):
-    def __init__(self, path: Path) -> None:
+    def __init__(self, path: Path, extras: List[str]) -> None:
         self.path = path
+        self.extras = extras
 
     def as_args(self) -> Iterable[str]:
-        return (str(self.path),)
+        return (str(self),)
 
     def __eq__(self, other: Any) -> bool:
-        return isinstance(other, self.__class__) and self.path == other.path
+        return isinstance(other, self.__class__) and self.path == other.path and self.extras == other.extras
 
     def __str__(self) -> str:
-        return str(self.path)
+        extra_group = f"[{','.join(self.extras)}]" if self.extras else ""
+        return f"{self.path}{extra_group}"
 
 
 class EditablePathReq(PathReq):
     def as_args(self) -> Iterable[str]:
-        return "-e", str(self.path)
+        return ("-e", super().__str__())
 
     def __str__(self) -> str:
-        return f"-e {self.path}"
+        return f"-e {super().__str__()}"
 
 
 class UrlReq(PipRequirementEntry):
@@ -143,6 +145,11 @@ class UrlReq(PipRequirementEntry):
         return self.url
 
 
+# https://www.python.org/dev/peps/pep-0508/#extras
+_EXTRA_PATH = re.compile(r"(.*)\[([-._,\sa-zA-Z0-9]*)]")
+_EXTRA_ELEMENT = re.compile(r"[a-zA-Z0-9]*[-._a-zA-Z0-9]")
+
+
 class PythonDeps:
     """A sub-set form of the requirements files (support tox 3 syntax, and --hash is not valid on CLI)"""
 
@@ -215,17 +222,35 @@ class PythonDeps:
             if is_url(line) or any(line.startswith(f"{v}+") and is_url(line[len(v) + 1 :]) for v in VCS):
                 result.append(UrlReq(line))
             else:
-                path = ini_dir / line
-                try:
-                    is_valid_file = path.exists() and (path.is_file() or path.is_dir())
-                except OSError:  # https://bugs.python.org/issue42855 # pragma: no cover
-                    is_valid_file = False  # pragma: no cover
-                if not is_valid_file:
+                for path, extra in self._path_candidate(ini_dir / line):
+                    try:
+                        if path.exists() and (path.is_file() or path.is_dir()):
+                            result.append(PathReq(path, extra))
+                            break
+                    except OSError:  # https://bugs.python.org/issue42855 # pragma: no cover
+                        continue
+                else:
                     raise ValueError(f"{at}: {line}") from exc
-                result.append(PathReq(path))
         else:
             result.append(req)
 
+    @staticmethod
+    def _path_candidate(path: Path) -> Iterator[Tuple[Path, List[str]]]:
+        yield path, []
+        # if there's a trailing [a,b] section that could mean either a folder or extras, try both
+        match = _EXTRA_PATH.fullmatch(path.name)
+        if match:
+            extras = []
+            for extra in match.group(2).split(","):
+                extra = extra.strip()
+                if not extra:
+                    continue
+                if not _EXTRA_ELEMENT.fullmatch(extra):
+                    break
+                extras.append(extra)
+            else:
+                yield path.parent / match.group(1), extras
+
     def _load_requirement_with_extra(self, line: str) -> Tuple[str, List[str]]:
         return line, []
 
@@ -254,7 +279,7 @@ class PythonDeps:
                 req_file.validate_and_expand()
                 result.append(req_file)
             elif first in ("-e", "--editable"):
-                result.append(EditablePathReq(Path(words[2])))
+                result.append(EditablePathReq(Path(words[2]), []))
             elif first in [
                 "-i",
                 "--index-url",
diff --git a/tests/tox_env/python/pip/test_req_file.py b/tests/tox_env/python/pip/test_req_file.py
index f73d020d..5f61f44f 100644
--- a/tests/tox_env/python/pip/test_req_file.py
+++ b/tests/tox_env/python/pip/test_req_file.py
@@ -45,6 +45,7 @@ from tox.tox_env.python.pip.req_file import (
         pytest.param("--extra-index-url a", "--extra-index-url a", id="extra-index-url"),
         pytest.param("-e a", "-e a", id="e"),
         pytest.param("--editable a", "-e a", id="editable"),
+        pytest.param("--editable .[extra1,extra2]", "-e .[extra1,extra2]", id="editable extra"),
         pytest.param("-f a", "-f a", id="f"),
         pytest.param("--find-links a", "--find-links a", id="find-links"),
         pytest.param("--trusted-host a", "--trusted-host a", id="trusted-host"),
@@ -138,6 +139,16 @@ def test_requirements_txt(tmp_path: Path, req: str, key: str) -> None:
         assert expanded == []
 
 
+def test_deps_path_with_extra_ok(tmp_path: Path) -> None:
+    result = PythonDeps(".[\t, a1. , B2-\t, C3_, ]", root=tmp_path).unroll()
+    assert result == [f"{tmp_path}[a1.,B2-,C3_]"]
+
+
+def test_deps_path_with_extra_nok(tmp_path: Path) -> None:
+    with pytest.raises(ValueError, match=re.escape(".[\t, a.1]")):
+        PythonDeps(".[\t, a.1]", root=tmp_path).unroll()
+
+
 def test_requirements_txt_local_path_file_protocol(tmp_path: Path) -> None:
     (tmp_path / "downloads").mkdir()
     (tmp_path / "downloads" / "numpy-1.9.2-cp34-none-win32.whl").write_text("1")
@@ -300,24 +311,24 @@ def test_requirement_with_flags_has_args() -> None:
 
 
 def test_path_req(tmp_path: Path) -> None:
-    path_req = PathReq(tmp_path)
+    path_req = PathReq(tmp_path, [])
 
     assert path_req.as_args() == (str(tmp_path),)
     assert str(path_req) == str(tmp_path)
 
-    assert path_req != PathReq(tmp_path / "a")
-    assert path_req == PathReq(tmp_path)
+    assert path_req != PathReq(tmp_path / "a", [])
+    assert path_req == PathReq(tmp_path, [])
     assert path_req != object
 
 
 def test_editable_path_req(tmp_path: Path) -> None:
-    editable_path_req = EditablePathReq(tmp_path)
+    editable_path_req = EditablePathReq(tmp_path, [])
 
     assert editable_path_req.as_args() == ("-e", str(tmp_path))
     assert str(editable_path_req) == f"-e {tmp_path}"
 
-    assert editable_path_req != EditablePathReq(tmp_path / "a")
-    assert editable_path_req == EditablePathReq(tmp_path)
+    assert editable_path_req != EditablePathReq(tmp_path / "a", [])
+    assert editable_path_req == EditablePathReq(tmp_path, [])
     assert editable_path_req != object