Windows embedable support (#2353)

* Bump pip and setuptools (#2348)

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

* Use shlex.quote instead of deprecated pipes.quote (#2351)

* Embeds the "python<VERSION>.zip" for Windows.

For example, for Python 3.10 the embeddable file name would be
"python310.zip". If this file would be found in `sys.path`, the
virtualenv should copy it into the "<venv>\Scripts\python310.zip".

* For Windows CPython3: *.dll/*.pyd -> to_bin

* Fixture for a Python interpreter info.

Helps to test virtualenv creator classes.

* Creators tests: path_mock as separate module.

* Clarifies tests, separates testing tools.

* Tests for CPython3Windows sources.

* Tests for the embedded Python std lib for Windows.

* Add news entry.

* Replaces `yield from` for backward compability.

* FIX: Path mocking in pypy tests.

* Wrap `sys` `Path` with `str` for importlib.

The importlib accepts a Path-like objects from Python 3.6

* Makes PathMock ABC compatible with Python 2

* Does not collect tests for Python3 under Python 2

It is possible to make pass CPython3 tests under Python 2,
but it's better to disable it instead of decreasing the
readability and performance of Python 3 style.

* Allows empty `Path()` in Windows with Python 2

* Allows to load fixture files with PY2 Windows Path

* Skips one PY3 POSIX test in PY2 Windows

Co-authored-by: Bernát Gábor <gaborjbernat@gmail.com>
Co-authored-by: Lumír 'Frenzy' Balhar <lbalhar@redhat.com>

8 files changed, 373 insertions, 91 deletions

diff --git a/tests/unit/create/via_global_ref/builtin/conftest.py b/tests/unit/create/via_global_ref/builtin/conftest.py
new file mode 100644
index 0000000..3c10636
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/conftest.py
@@ -0,0 +1,25 @@
+import sys
+
+import pytest
+from testing import path
+from testing.py_info import read_fixture
+
+from virtualenv.util.path import Path
+
+# Allows to import from `testing` into test submodules.
+sys.path.append(str(Path(__file__).parent))
+
+
+@pytest.fixture
+def py_info(py_info_name):
+    return read_fixture(py_info_name)
+
+
+@pytest.fixture
+def mock_files(mocker):
+    return lambda paths, files: path.mock_files(mocker, paths, files)
+
+
+@pytest.fixture
+def mock_pypy_libs(mocker):
+    return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs)
diff --git a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json
new file mode 100644
index 0000000..e8d0d01
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json
@@ -0,0 +1,61 @@
+{
+  "platform": "win32",
+  "implementation": "CPython",
+  "version_info": {
+    "major": 3,
+    "minor": 10,
+    "micro": 4,
+    "releaselevel": "final",
+    "serial": 0
+  },
+  "architecture": 64,
+  "version_nodot": "310",
+  "version": "3.10.4 (tags/v3.10.4:9d38120, Mar 23 2022, 23:13:41) [MSC v.1929 64 bit (AMD64)]",
+  "os": "nt",
+  "prefix": "c:\\path\\to\\python",
+  "base_prefix": "c:\\path\\to\\python",
+  "real_prefix": null,
+  "base_exec_prefix": "c:\\path\\to\\python",
+  "exec_prefix": "c:\\path\\to\\python",
+  "executable": "c:\\path\\to\\python\\python.exe",
+  "original_executable": "c:\\path\\to\\python\\python.exe",
+  "system_executable": "c:\\path\\to\\python\\python.exe",
+  "has_venv": false,
+  "path": [
+    "c:\\path\\to\\python\\Scripts\\virtualenv.exe",
+    "c:\\path\\to\\python\\python310.zip",
+    "c:\\path\\to\\python",
+    "c:\\path\\to\\python\\Lib\\site-packages"
+  ],
+  "file_system_encoding": "utf-8",
+  "stdout_encoding": "utf-8",
+  "sysconfig_scheme": null,
+  "sysconfig_paths": {
+    "stdlib": "{installed_base}/Lib",
+    "platstdlib": "{base}/Lib",
+    "purelib": "{base}/Lib/site-packages",
+    "platlib": "{base}/Lib/site-packages",
+    "include": "{installed_base}/Include",
+    "scripts": "{base}/Scripts",
+    "data": "{base}"
+  },
+  "distutils_install": {
+    "purelib": "Lib\\site-packages",
+    "platlib": "Lib\\site-packages",
+    "headers": "Include\\UNKNOWN",
+    "scripts": "Scripts",
+    "data": ""
+  },
+  "sysconfig": {
+    "makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile"
+  },
+  "sysconfig_vars": {
+    "PYTHONFRAMEWORK": "",
+    "installed_base": "c:\\path\\to\\python",
+    "base": "c:\\path\\to\\python"
+  },
+  "system_stdlib": "c:\\path\\to\\python\\Lib",
+  "system_stdlib_platform": "c:\\path\\to\\python\\Lib",
+  "max_size": 9223372036854775807,
+  "_creators": null
+}
diff --git a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py
new file mode 100644
index 0000000..087d590
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py
@@ -0,0 +1,99 @@
+import pytest
+from testing.helpers import contains_exe, contains_ref
+from testing.path import join as path
+
+from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Windows
+
+CPYTHON3_PATH = (
+    "virtualenv.create.via_global_ref.builtin.cpython.common.Path",
+    "virtualenv.create.via_global_ref.builtin.cpython.cpython3.Path",
+)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_2_exe_on_default_py_host(py_info, mock_files):
+    mock_files(CPYTHON3_PATH, [py_info.system_executable])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    # Default Python exe.
+    assert contains_exe(sources, py_info.system_executable)
+    # Should always exist.
+    assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_3_exe_on_not_default_py_host(py_info, mock_files):
+    # Not default python host.
+    py_info.system_executable = path(py_info.prefix, "python666.exe")
+    mock_files(CPYTHON3_PATH, [py_info.system_executable])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    # Not default Python exe linked to both the default name and origin.
+    assert contains_exe(sources, py_info.system_executable, "python.exe")
+    assert contains_exe(sources, py_info.system_executable, "python666.exe")
+    # Should always exist.
+    assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_only_shim(py_info, mock_files):
+    shim = path(py_info.system_stdlib, "venv\\scripts\\nt\\python.exe")
+    py_files = (
+        path(py_info.prefix, "libcrypto-1_1.dll"),
+        path(py_info.prefix, "libffi-7.dll"),
+        path(py_info.prefix, "_asyncio.pyd"),
+        path(py_info.prefix, "_bz2.pyd"),
+    )
+    mock_files(CPYTHON3_PATH, [shim, *py_files])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert CPython3Windows.has_shim(interpreter=py_info)
+    assert contains_exe(sources, shim)
+    assert not contains_exe(sources, py_info.system_executable)
+    for file in py_files:
+        assert not contains_ref(sources, file)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_exe_dll_pyd_without_shim(py_info, mock_files):
+    py_files = (
+        path(py_info.prefix, "libcrypto-1_1.dll"),
+        path(py_info.prefix, "libffi-7.dll"),
+        path(py_info.prefix, "_asyncio.pyd"),
+        path(py_info.prefix, "_bz2.pyd"),
+    )
+    mock_files(CPYTHON3_PATH, py_files)
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert not CPython3Windows.has_shim(interpreter=py_info)
+    assert contains_exe(sources, py_info.system_executable)
+    for file in py_files:
+        assert contains_ref(sources, file)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_python_zip_if_exists_and_set_in_path(py_info, mock_files):
+    python_zip_name = "python{}.zip".format(py_info.version_nodot)
+    python_zip = path(py_info.prefix, python_zip_name)
+    mock_files(CPYTHON3_PATH, [python_zip])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert python_zip in py_info.path
+    assert contains_ref(sources, python_zip)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files):
+    python_zip_name = "python{}.zip".format(py_info.version_nodot)
+    python_zip = path(py_info.prefix, python_zip_name)
+    py_info.path.remove(python_zip)
+    mock_files(CPYTHON3_PATH, [python_zip])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert python_zip not in py_info.path
+    assert not contains_ref(sources, python_zip)
+
+
+@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
+def test_no_python_zip_if_not_exists(py_info, mock_files):
+    python_zip_name = "python{}.zip".format(py_info.version_nodot)
+    python_zip = path(py_info.prefix, python_zip_name)
+    # No `python_zip`, just python.exe file.
+    mock_files(CPYTHON3_PATH, [py_info.system_executable])
+    sources = tuple(CPython3Windows.sources(interpreter=py_info))
+    assert python_zip in py_info.path
+    assert not contains_ref(sources, python_zip)
diff --git a/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py b/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py
index c4d6860..eb35d7f 100644
--- a/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py
+++ b/tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py
@@ -1,104 +1,51 @@
 from __future__ import absolute_import, unicode_literals
 
-import fnmatch
+import pytest
+from testing.helpers import contains_exe, contains_ref
+from testing.path import join as path
 
 from virtualenv.create.via_global_ref.builtin.pypy.pypy3 import PyPy3Posix
-from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest
-from virtualenv.discovery.py_info import PythonInfo
-from virtualenv.util.path import Path
-
-
-class FakePath(Path):
-    """
-    A Path() fake that only knows about files in existing_paths and the
-    directories that contain them.
-    """
-
-    existing_paths = []
-
-    if hasattr(Path(""), "_flavour"):
-        _flavour = Path("")._flavour
-
-    def exists(self):
-        return self.as_posix() in self.existing_paths or self.is_dir()
-
-    def glob(self, glob):
-        pattern = self.as_posix() + "/" + glob
-        for path in fnmatch.filter(self.existing_paths, pattern):
-            yield FakePath(path)
-
-    def is_dir(self):
-        prefix = self.as_posix() + "/"
-        return any(True for path in self.existing_paths if path.startswith(prefix))
-
-    def iterdir(self):
-        prefix = self.as_posix() + "/"
-        for path in self.existing_paths:
-            if path.startswith(prefix) and "/" not in path[len(prefix) :]:
-                yield FakePath(path)
-
-    def resolve(self):
-        return self
-
-    def __div__(self, key):
-        return FakePath(super(FakePath, self).__div__(key))
-
-    def __truediv__(self, key):
-        return FakePath(super(FakePath, self).__truediv__(key))
-
-
-def assert_contains_exe(sources, src):
-    """Assert that the one and only executeable in sources is src"""
-    exes = [source for source in sources if isinstance(source, ExePathRefToDest)]
-    assert len(exes) == 1
-    exe = exes[0]
-    assert exe.src.as_posix() == src
-
-
-def assert_contains_ref(sources, src):
-    """Assert that src appears in sources"""
-    assert any(source for source in sources if isinstance(source, PathRefToDest) and source.src.as_posix() == src)
-
-
-def inject_fake_path(mocker, existing_paths):
-    """Inject FakePath in all the correct places, and set existing_paths"""
-    FakePath.existing_paths = existing_paths
-    mocker.patch("virtualenv.create.via_global_ref.builtin.pypy.common.Path", FakePath)
-    mocker.patch("virtualenv.create.via_global_ref.builtin.pypy.pypy3.Path", FakePath)
-
-
-def _load_pypi_info(name):
-    return PythonInfo._from_json((Path(__file__).parent / "{}.json".format(name)).read_text())
-
-
-def test_portable_pypy3_virtualenvs_get_their_libs(mocker):
-    paths = ["/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", "/tmp/pypy3.8-v7.3.8-linux64/lib/libgdbm.so.4"]
-    inject_fake_path(mocker, paths)
-    path = Path("/tmp/pypy3.8-v7.3.8-linux64/bin/libpypy3-c.so")
-    mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[path])
-
-    sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("portable_pypy38")))
-    assert_contains_exe(sources, "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy")
+from virtualenv.info import IS_WIN, PY2
+
+PYPY3_PATH = (
+    "virtualenv.create.via_global_ref.builtin.pypy.common.Path",
+    "virtualenv.create.via_global_ref.builtin.pypy.pypy3.Path",
+)
+
+
+# In `PyPy3Posix.sources()` `host_lib` will be broken in Python 2 for Windows,
+# so `py_file` will not be in sources.
+@pytest.mark.skipif(PY2 and IS_WIN, reason="Can't convert PosixPath")
+@pytest.mark.parametrize("py_info_name", ["portable_pypy38"])
+def test_portable_pypy3_virtualenvs_get_their_libs(py_info, mock_files, mock_pypy_libs):
+    py_file = path(py_info.prefix, "lib/libgdbm.so.4")
+    mock_files(PYPY3_PATH, [py_info.system_executable, py_file])
+    lib_file = path(py_info.prefix, "bin/libpypy3-c.so")
+    mock_pypy_libs(PyPy3Posix, [lib_file])
+    sources = tuple(PyPy3Posix.sources(interpreter=py_info))
     assert len(sources) > 2
-    assert_contains_ref(sources, "/tmp/pypy3.8-v7.3.8-linux64/bin/libpypy3-c.so")
-    assert_contains_ref(sources, "/tmp/pypy3.8-v7.3.8-linux64/lib/libgdbm.so.4")
+    assert contains_exe(sources, py_info.system_executable)
+    assert contains_ref(sources, py_file)
+    assert contains_ref(sources, lib_file)
 
 
-def test_debian_pypy37_virtualenvs(mocker):
+@pytest.mark.parametrize("py_info_name", ["deb_pypy37"])
+def test_debian_pypy37_virtualenvs(py_info, mock_files, mock_pypy_libs):
     # Debian's pypy3 layout, installed to /usr, before 3.8 allowed a /usr prefix
-    inject_fake_path(mocker, ["/usr/bin/pypy3"])
-    mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[Path("/usr/lib/pypy3/bin/libpypy3-c.so")])
-    sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("deb_pypy37")))
-    assert_contains_exe(sources, "/usr/bin/pypy3")
-    assert_contains_ref(sources, "/usr/lib/pypy3/bin/libpypy3-c.so")
+    mock_files(PYPY3_PATH, [py_info.system_executable])
+    lib_file = path(py_info.prefix, "bin/libpypy3-c.so")
+    mock_pypy_libs(PyPy3Posix, [lib_file])
+    sources = tuple(PyPy3Posix.sources(interpreter=py_info))
     assert len(sources) == 2
+    assert contains_exe(sources, py_info.system_executable)
+    assert contains_ref(sources, lib_file)
 
 
-def test_debian_pypy38_virtualenvs_exclude_usr(mocker):
-    inject_fake_path(mocker, ["/usr/bin/pypy3", "/usr/lib/foo"])
+@pytest.mark.parametrize("py_info_name", ["deb_pypy38"])
+def test_debian_pypy38_virtualenvs_exclude_usr(py_info, mock_files, mock_pypy_libs):
+    mock_files(PYPY3_PATH, [py_info.system_executable, "/usr/lib/foo"])
     # libpypy3-c.so lives on the ld search path
-    mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[])
-
-    sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("deb_pypy38")))
-    assert_contains_exe(sources, "/usr/bin/pypy3")
+    mock_pypy_libs(PyPy3Posix, [])
+    sources = tuple(PyPy3Posix.sources(interpreter=py_info))
     assert len(sources) == 1
+    assert contains_exe(sources, py_info.system_executable)
diff --git a/tests/unit/create/via_global_ref/builtin/testing/__init__.py b/tests/unit/create/via_global_ref/builtin/testing/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/testing/__init__.py
diff --git a/tests/unit/create/via_global_ref/builtin/testing/helpers.py b/tests/unit/create/via_global_ref/builtin/testing/helpers.py
new file mode 100644
index 0000000..c42dd39
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/testing/helpers.py
@@ -0,0 +1,38 @@
+from functools import reduce
+
+from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRef
+from virtualenv.util.path import Path
+
+
+def is_ref(source):
+    return isinstance(source, PathRef)
+
+
+def is_exe(source):
+    return type(source) is ExePathRefToDest
+
+
+def has_src(src):
+    return lambda ref: ref.src.as_posix() == Path(src).as_posix()
+
+
+def has_target(target):
+    return lambda ref: ref.base == target
+
+
+def apply_filter(values, function):
+    return filter(function, values)
+
+
+def filterby(filters, sources):
+    return reduce(apply_filter, filters, sources)
+
+
+def contains_exe(sources, src, target=None):
+    filters = is_exe, has_src(src), target and has_target(target)
+    return any(filterby(filters, sources))
+
+
+def contains_ref(sources, src):
+    filters = is_ref, has_src(src)
+    return any(filterby(filters, sources))
diff --git a/tests/unit/create/via_global_ref/builtin/testing/path.py b/tests/unit/create/via_global_ref/builtin/testing/path.py
new file mode 100644
index 0000000..ece41cf
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/testing/path.py
@@ -0,0 +1,93 @@
+from abc import ABCMeta, abstractmethod
+from itertools import chain
+from operator import attrgetter as attr
+
+from six import add_metaclass
+
+from virtualenv.util.path import Path
+
+
+def is_name(path):
+    return str(path) == path.name
+
+
+@add_metaclass(ABCMeta)
+class FakeDataABC(object):
+    """Provides data to mock the `Path`"""
+
+    @property
+    @abstractmethod
+    def filelist(self):
+        raise NotImplementedError("Collection of (str) file paths to mock")
+
+    @property
+    def fake_files(self):
+        return map(type(self), self.filelist)
+
+    @property
+    def fake_dirs(self):
+        return set(chain(*map(attr("parents"), self.fake_files)))
+
+    @property
+    def contained_fake_names(self):
+        return filter(is_name, self.fake_content)
+
+    @property
+    def fake_content(self):
+        return filter(None, map(self.fake_child, self.fake_files))
+
+    def fake_child(self, path):
+        try:
+            return path.relative_to(self)
+        except ValueError:
+            return None
+
+
+class PathMockABC(FakeDataABC, Path):
+    """Mocks the behavior of `Path`"""
+
+    _flavour = getattr(Path(), "_flavour", None)
+
+    if hasattr(_flavour, "altsep"):
+        # Allows to pass some tests for Windows via PosixPath.
+        _flavour.altsep = _flavour.altsep or "\\"
+
+    def exists(self):
+        return self.is_file() or self.is_dir()
+
+    def is_file(self):
+        return self in self.fake_files
+
+    def is_dir(self):
+        return self in self.fake_dirs
+
+    def resolve(self):
+        return self
+
+    def iterdir(self):
+        for path in map(self.joinpath, self.contained_fake_names):
+            yield path
+
+
+def MetaPathMock(filelist):
+    """
+    Metaclass that creates a `PathMock` class with the `filelist` defined.
+    """
+    return type("PathMock", (PathMockABC,), {"filelist": filelist})
+
+
+def mock_files(mocker, pathlist, filelist):
+    PathMock = MetaPathMock(set(filelist))
+    for path in pathlist:
+        mocker.patch(path, PathMock)
+
+
+def mock_pypy_libs(mocker, pypy_creator_cls, libs):
+    paths = tuple(set(map(Path, libs)))
+    mocker.patch.object(pypy_creator_cls, "_shared_libs", return_value=paths)
+
+
+def join(*chunks):
+    line = "".join(chunks)
+    sep = ("\\" in line and "\\") or ("/" in line and "/") or "/"
+    return sep.join(chunks)
diff --git a/tests/unit/create/via_global_ref/builtin/testing/py_info.py b/tests/unit/create/via_global_ref/builtin/testing/py_info.py
new file mode 100644
index 0000000..20dafb6
--- /dev/null
+++ b/tests/unit/create/via_global_ref/builtin/testing/py_info.py
@@ -0,0 +1,19 @@
+from virtualenv.discovery.py_info import PythonInfo
+from virtualenv.info import PY2
+from virtualenv.util.path import Path
+
+
+def fixture_file(fixture_name):
+    file_mask = "*{}.json".format(fixture_name)
+    files = Path(__file__).parent.parent.rglob(file_mask)
+    try:
+        return next(files)
+    except StopIteration:
+        # Fixture file was not found in the testing root and its subdirs.
+        error = NameError if PY2 else FileNotFoundError
+        raise error(file_mask)
+
+
+def read_fixture(fixture_name):
+    fixture_json = fixture_file(fixture_name).read_text()
+    return PythonInfo._from_json(fixture_json)