Cross version compatibility (#1475)

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

4 files changed, 120 insertions, 31 deletions

diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py
index b86f11e..1dc80c0 100644
--- a/src/virtualenv/interpreters/discovery/builtin.py
+++ b/src/virtualenv/interpreters/discovery/builtin.py
@@ -1,8 +1,10 @@
 from __future__ import absolute_import, unicode_literals
 
+import logging
 import os
 import sys
-from distutils.spawn import find_executable
+
+from pathlib2 import Path
 
 from virtualenv.info import IS_WIN
 
@@ -36,16 +38,21 @@ class Builtin(Discover):
 
 def get_interpreter(key):
     spec = PythonSpec.from_string_spec(key)
+    logging.debug("find interpreter for spec %r", spec)
+    proposed_paths = set()
     for interpreter, impl_must_match in propose_interpreters(spec):
-        if interpreter.satisfies(spec, impl_must_match):
-            return interpreter
+        if interpreter.executable not in proposed_paths:
+            logging.debug("proposed %s", interpreter)
+            if interpreter.satisfies(spec, impl_must_match):
+                return interpreter
+            proposed_paths.add(interpreter.executable)
 
 
 def propose_interpreters(spec):
     # 1. we always try with the lowest hanging fruit first, the current interpreter
     yield CURRENT, True
 
-    # 2. if it's an absolut path and exists, use that
+    # 2. if it's an absolute path and exists, use that
     if spec.is_abs and os.path.exists(spec.path):
         yield PythonInfo.from_exe(spec.path), True
 
@@ -56,21 +63,75 @@ def propose_interpreters(spec):
         for interpreter in propose_interpreters(spec):
             yield interpreter, True
 
-    # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts
-    interpreter = find_on_path(spec.str_spec)
-    if interpreter is not None:
-        yield interpreter, False
+    paths = get_paths()
+    for path in paths:  # find on path, the path order matters (as the candidates are less easy to control by end user)
+        for candidate, match in possible_specs(spec):
+            found = check_path(candidate, path)
+            if found is not None:
+                exe = os.path.abspath(found)
+                interpreter = PathPythonInfo.from_exe(exe, raise_on_error=False)
+                if interpreter is not None:
+                    yield interpreter, match
+
+
+def get_paths():
+    path = os.environ.get(str("PATH"), None)
+    if path is None:
+        try:
+            path = os.confstr("CS_PATH")
+        except (AttributeError, ValueError):
+            path = os.defpath
+    if not path:
+        paths = []
+    else:
+        paths = [p for p in path.split(os.pathsep) if os.path.exists(p)]
+    logging.debug(LazyPathDump(paths))
+    return paths
+
+
+class LazyPathDump(object):
+    def __init__(self, paths):
+        self.paths = paths
 
+    def __str__(self):
+        content = "PATH =>{}".format(os.linesep)
+        for i, p in enumerate(self.paths):
+            files = []
+            for file in Path(p).iterdir():
+                try:
+                    if file.is_dir():
+                        continue
+                except OSError:
+                    pass
+                files.append(file.name)
+            content += str(i)
+            content += " "
+            content += str(p)
+            content += " with "
+            content += " ".join(files)
+            content += os.linesep
+        return content
+
+
+def check_path(candidate, path):
+    _, ext = os.path.splitext(candidate)
+    if sys.platform == "win32" and ext != ".exe":
+        candidate = candidate + ".exe"
+    if os.path.isfile(candidate):
+        return candidate
+    candidate = os.path.join(path, candidate)
+    if os.path.isfile(candidate):
+        return candidate
+    return None
+
+
+def possible_specs(spec):
+    # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts
+    yield spec.str_spec, False
     # 5. or from the spec we can deduce a name on path  that matches
     for exe, match in spec.generate_names():
-        interpreter = find_on_path(exe)
-        if interpreter is not None:
-            yield interpreter, match
+        yield exe, match
 
 
-def find_on_path(key):
-    exe = find_executable(key)
-    if exe is not None:
-        exe = os.path.abspath(exe)
-        interpreter = PythonInfo.from_exe(str(exe), raise_on_error=False)
-        return interpreter
+class PathPythonInfo(PythonInfo):
+    """"""
diff --git a/src/virtualenv/interpreters/discovery/py_info.py b/src/virtualenv/interpreters/discovery/py_info.py
index 824a2be..2af2394 100644
--- a/src/virtualenv/interpreters/discovery/py_info.py
+++ b/src/virtualenv/interpreters/discovery/py_info.py
@@ -84,13 +84,34 @@ class PythonInfo(object):
         return self.base_prefix is not None and self.version_info.major == 3
 
     def __repr__(self):
-        return "PythonInfo({!r})".format(self.__dict__)
+        return "{}({!r})".format(self.__class__.__name__, self.__dict__)
 
     def __str__(self):
-        content = copy.copy(self.__dict__)
-        for elem in ["path", "prefix", "base_prefix", "exec_prefix", "real_prefix", "base_exec_prefix"]:
-            del content[elem]
-        return "PythonInfo({!r})".format(content)
+        return "{}({})".format(
+            self.__class__.__name__,
+            ", ".join(
+                "{}={}".format(k, v)
+                for k, v in (
+                    (
+                        "spec",
+                        "{}{}-{}".format(
+                            self.implementation, ".".join(str(i) for i in self.version_info), self.architecture
+                        ),
+                    ),
+                    ("exe", self.executable),
+                    ("original" if self.original_executable != self.executable else None, self.original_executable),
+                    (
+                        "base"
+                        if self.base_executable is not None and self.base_executable != self.executable
+                        else None,
+                        self.base_executable,
+                    ),
+                    ("platform", self.platform),
+                    ("version", repr(self.version)),
+                )
+                if k is not None
+            ),
+        )
 
     def to_json(self):
         data = copy.deepcopy(self.__dict__)
@@ -117,12 +138,15 @@ class PythonInfo(object):
     @property
     def system_executable(self):
         env_prefix = self.real_prefix or self.base_prefix
-        if env_prefix:
-            if self.real_prefix is None and self.base_executable is not None:
+        if env_prefix:  # if this is a virtual environment
+            if self.real_prefix is None and self.base_executable is not None:  # use the saved host if present
                 return self.base_executable
+            # otherwise fallback to discovery mechanism
             return self.find_exe_based_of(inside_folder=env_prefix)
         else:
-            return self.executable
+            # need original executable here, as if we need to copy we want to copy the interpreter itself, not the
+            # setup script things may be wrapped up in
+            return self.original_executable
 
     def find_exe_based_of(self, inside_folder):
         # we don't know explicitly here, do some guess work - our executable name should tell
@@ -162,16 +186,16 @@ class PythonInfo(object):
                         name_candidate[candidate] = None
         return list(name_candidate.keys())
 
-    __cache_from_exe = {}
+    _cache_from_exe = {}
 
     @classmethod
     def from_exe(cls, exe, raise_on_error=True):
         key = os.path.realpath(exe)
-        if key in cls.__cache_from_exe:
-            result, failure = cls.__cache_from_exe[key]
+        if key in cls._cache_from_exe:
+            result, failure = cls._cache_from_exe[key]
         else:
             failure, result = cls._load_for_exe(exe)
-            cls.__cache_from_exe[key] = result, failure
+            cls._cache_from_exe[key] = result, failure
         if failure is not None:
             if raise_on_error:
                 raise failure
diff --git a/src/virtualenv/interpreters/discovery/py_spec.py b/src/virtualenv/interpreters/discovery/py_spec.py
index 382bf2e..ab08491 100644
--- a/src/virtualenv/interpreters/discovery/py_spec.py
+++ b/src/virtualenv/interpreters/discovery/py_spec.py
@@ -6,7 +6,7 @@ import re
 import sys
 from collections import OrderedDict
 
-PATTERN = re.compile(r"^(?P<impl>[a-zA-Z]+)(?P<version>[0-9.]+)?(?:-(?P<arch>32|64))?$")
+PATTERN = re.compile(r"^(?P<impl>[a-zA-Z]+)?(?P<version>[0-9.]+)?(?:-(?P<arch>32|64))?$")
 IS_WIN = sys.platform == "win32"
 
 
diff --git a/src/virtualenv/interpreters/discovery/windows/__init__.py b/src/virtualenv/interpreters/discovery/windows/__init__.py
index 42ec221..f321d7f 100644
--- a/src/virtualenv/interpreters/discovery/windows/__init__.py
+++ b/src/virtualenv/interpreters/discovery/windows/__init__.py
@@ -5,13 +5,17 @@ from ..py_spec import PythonSpec
 from .pep514 import discover_pythons
 
 
+class Pep514PythonInfo(PythonInfo):
+    """"""
+
+
 def propose_interpreters(spec):
     # see if PEP-514 entries are good
     for name, major, minor, arch, exe, _ in discover_pythons():
         # pre-filter
         registry_spec = PythonSpec(None, name, major, minor, None, arch, exe)
         if registry_spec.satisfies(spec):
-            interpreter = PythonInfo.from_exe(exe, raise_on_error=False)
+            interpreter = Pep514PythonInfo.from_exe(exe, raise_on_error=False)
             if interpreter is not None:
                 if interpreter.satisfies(spec, impl_must_match=True):
                     yield interpreter