unicode support (#1477)

* creator unicode support

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* activator support

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* fix

* add space

* python3.4 support

* Windows fixes

* some fixes

* fix powershell requires utf-16

* try to fix python2 windows

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* use utf-8 for activation scripts

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* fix

* more fix

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* fix

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* windows path py2.7

* fixes for Python 2 and unicode on Windows

* do not single out mbcs, but the file system encoder

* do not install pathlib python 2 windows

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

* fix encoding on py35

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

8 files changed, 99 insertions, 68 deletions

diff --git a/src/virtualenv/interpreters/create/cpython/common.py b/src/virtualenv/interpreters/create/cpython/common.py
index d05d47b..89a91c6 100644
--- a/src/virtualenv/interpreters/create/cpython/common.py
+++ b/src/virtualenv/interpreters/create/cpython/common.py
@@ -4,10 +4,9 @@ import abc
 from os import X_OK, access, chmod
 
 import six
-from pathlib2 import Path
 
 from virtualenv.interpreters.create.via_global_ref import ViaGlobalRef
-from virtualenv.util import copy, ensure_dir, symlink
+from virtualenv.util import Path, copy, ensure_dir, symlink
 
 
 @six.add_metaclass(abc.ABCMeta)
diff --git a/src/virtualenv/interpreters/create/cpython/cpython2.py b/src/virtualenv/interpreters/create/cpython/cpython2.py
index fbaee88..7c79158 100644
--- a/src/virtualenv/interpreters/create/cpython/cpython2.py
+++ b/src/virtualenv/interpreters/create/cpython/cpython2.py
@@ -3,9 +3,8 @@ from __future__ import absolute_import, unicode_literals
 import abc
 
 import six
-from pathlib2 import Path
 
-from virtualenv.util import copy
+from virtualenv.util import Path, copy
 
 from .common import CPython, CPythonPosix, CPythonWindows
 
diff --git a/src/virtualenv/interpreters/create/cpython/cpython3.py b/src/virtualenv/interpreters/create/cpython/cpython3.py
index 3249306..b14ce0a 100644
--- a/src/virtualenv/interpreters/create/cpython/cpython3.py
+++ b/src/virtualenv/interpreters/create/cpython/cpython3.py
@@ -3,9 +3,8 @@ from __future__ import absolute_import, unicode_literals
 import abc
 
 import six
-from pathlib2 import Path
 
-from virtualenv.util import copy
+from virtualenv.util import Path, copy
 
 from .common import CPython, CPythonPosix, CPythonWindows
 
diff --git a/src/virtualenv/interpreters/create/creator.py b/src/virtualenv/interpreters/create/creator.py
index 872d8d1..b917a37 100644
--- a/src/virtualenv/interpreters/create/creator.py
+++ b/src/virtualenv/interpreters/create/creator.py
@@ -4,15 +4,16 @@ import json
 import logging
 import os
 import shutil
+import sys
 from abc import ABCMeta, abstractmethod
 from argparse import ArgumentTypeError
 
-from pathlib2 import Path
+import six
 from six import add_metaclass
 
 from virtualenv.info import IS_WIN
 from virtualenv.pyenv_cfg import PyEnvCfg
-from virtualenv.util import run_cmd
+from virtualenv.util import Path, run_cmd
 from virtualenv.version import __version__
 
 HERE = Path(__file__).absolute().parent
@@ -47,28 +48,13 @@ class Creator(object):
             help="Give the virtual environment access to the system site-packages dir.",
         )
 
-        def validate_dest_dir(value):
-            """No path separator in the path and must be write-able"""
-            if os.pathsep in value:
-                raise ArgumentTypeError(
-                    "destination {!r} must not contain the path separator ({}) as this would break "
-                    "the activation scripts".format(value, os.pathsep)
-                )
-            value = Path(value)
-            if value.exists() and value.is_file():
-                raise ArgumentTypeError("the destination {} already exists and is a file".format(value))
-            value = dest = value.resolve()
-            while dest:
-                if dest.exists():
-                    if os.access(str(dest), os.W_OK):
-                        break
-                    else:
-                        non_write_able(dest, value)
-                base, _ = dest.parent, dest.name
-                if base == dest:
-                    non_write_able(dest, value)  # pragma: no cover
-                dest = base
-            return str(value)
+        parser.add_argument(
+            "dest_dir", help="directory to create virtualenv at", type=cls.validate_dest_dir, default="env", nargs="?",
+        )
+
+    @classmethod
+    def validate_dest_dir(cls, raw_value):
+        """No path separator in the path, valid chars and must be write-able"""
 
         def non_write_able(dest, value):
             common = Path(*os.path.commonprefix([value.parts, dest.parts]))
@@ -76,9 +62,45 @@ class Creator(object):
                 "the destination {} is not write-able at {}".format(dest.relative_to(common), common)
             )
 
-        parser.add_argument(
-            "dest_dir", help="directory to create virtualenv at", type=validate_dest_dir, default="env", nargs="?",
-        )
+        # the file system must be able to encode
+        # note in newer CPython this is always utf-8 https://www.python.org/dev/peps/pep-0529/
+        encoding = sys.getfilesystemencoding()
+        path_converted = raw_value.encode(encoding, errors="ignore").decode(encoding)
+        if path_converted != raw_value:
+            refused = set(raw_value) - {
+                c
+                for c, i in ((char, char.encode(encoding)) for char in raw_value)
+                if c == "?" or i != six.ensure_str("?")
+            }
+            raise ArgumentTypeError(
+                "the file system codec ({}) does not support characters {!r}".format(encoding, refused)
+            )
+        if os.pathsep in raw_value:
+            raise ArgumentTypeError(
+                "destination {!r} must not contain the path separator ({}) as this would break "
+                "the activation scripts".format(raw_value, os.pathsep)
+            )
+
+        value = Path(raw_value)
+        if value.exists() and value.is_file():
+            raise ArgumentTypeError("the destination {} already exists and is a file".format(value))
+        if (3, 3) <= sys.version_info <= (3, 6):
+            # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation
+            dest = Path(os.path.realpath(raw_value))
+        else:
+            dest = value.resolve()
+        value = dest
+        while dest:
+            if dest.exists():
+                if os.access(six.ensure_text(str(dest)), os.W_OK):
+                    break
+                else:
+                    non_write_able(dest, value)
+            base, _ = dest.parent, dest.name
+            if base == dest:
+                non_write_able(dest, value)  # pragma: no cover
+            dest = base
+        return str(value)
 
     def run(self):
         if self.dest_dir.exists() and self.clear:
@@ -104,7 +126,7 @@ class Creator(object):
 
     @property
     def env_name(self):
-        return self.dest_dir.parts[-1]
+        return six.ensure_text(self.dest_dir.parts[-1])
 
     @property
     def bin_name(self):
@@ -138,8 +160,8 @@ class Creator(object):
 
 
 def get_env_debug_info(env_exe, debug_script):
-    cmd = [str(env_exe), str(debug_script)]
-    logging.debug(" ".join(cmd))
+    cmd = [six.ensure_text(str(env_exe)), six.ensure_text(str(debug_script))]
+    logging.debug(" ".join(six.ensure_text(i) for i in cmd))
     env = os.environ.copy()
     env.pop("PYTHONPATH", None)
     code, out, err = run_cmd(cmd)
diff --git a/src/virtualenv/interpreters/create/debug.py b/src/virtualenv/interpreters/create/debug.py
index cb4da8e..37f1a45 100644
--- a/src/virtualenv/interpreters/create/debug.py
+++ b/src/virtualenv/interpreters/create/debug.py
@@ -2,6 +2,20 @@
 import sys  # built-in
 
 
+def encode_path(value):
+    if value is None:
+        return None
+    if isinstance(value, bytes):
+        return value.decode(sys.getfilesystemencoding())
+    if isinstance(value, type):
+        return repr(value)
+    return value
+
+
+def encode_list_path(value):
+    return [encode_path(i) for i in value]
+
+
 def run():
     """print debug data about the virtual environment"""
     try:
@@ -11,7 +25,7 @@ def run():
         # noinspection PyPep8Naming
         OrderedDict = dict  # pragma: no cover
     result = OrderedDict([("sys", OrderedDict())])
-    for key in (
+    path_keys = (
         "executable",
         "_base_executable",
         "prefix",
@@ -21,13 +35,15 @@ def run():
         "base_exec_prefix",
         "path",
         "meta_path",
-        "version",
-    ):
+    )
+    for key in path_keys:
         value = getattr(sys, key, None)
-        if key == "meta_path" and value is not None:
-            value = [repr(i) for i in value]
+        if isinstance(value, list):
+            value = encode_list_path(value)
+        else:
+            value = encode_path(value)
         result["sys"][key] = value
-
+    result["version"] = sys.version
     import os  # landmark
 
     result["os"] = os.__file__
@@ -45,7 +61,7 @@ def run():
 
         result["json"] = repr(json)
         print(json.dumps(result, indent=2))
-    except ImportError as exception:  # pragma: no cover
+    except (ImportError, ValueError, TypeError) as exception:  # pragma: no cover
         result["json"] = repr(exception)  # pragma: no cover
         print(repr(result))  # pragma: no cover
         raise SystemExit(1)  # pragma: no cover
diff --git a/src/virtualenv/interpreters/create/venv.py b/src/virtualenv/interpreters/create/venv.py
index 6c71d94..e80a45e 100644
--- a/src/virtualenv/interpreters/create/venv.py
+++ b/src/virtualenv/interpreters/create/venv.py
@@ -47,7 +47,7 @@ class Venv(ViaGlobalRef):
             raise ProcessCallFailed(code, out, err, cmd)
 
     def get_host_create_cmd(self):
-        cmd = [str(self.interpreter.system_executable), "-m", "venv", "--without-pip"]
+        cmd = [self.interpreter.system_executable, "-m", "venv", "--without-pip"]
         if self.system_site_package:
             cmd.append("--system-site-packages")
         cmd.append("--symlinks" if self.symlinks else "--copies")
diff --git a/src/virtualenv/interpreters/discovery/builtin.py b/src/virtualenv/interpreters/discovery/builtin.py
index 1dc80c0..166066d 100644
--- a/src/virtualenv/interpreters/discovery/builtin.py
+++ b/src/virtualenv/interpreters/discovery/builtin.py
@@ -4,7 +4,7 @@ import logging
 import os
 import sys
 
-from pathlib2 import Path
+import six
 
 from virtualenv.info import IS_WIN
 
@@ -64,7 +64,10 @@ def propose_interpreters(spec):
             yield interpreter, True
 
     paths = get_paths()
-    for path in paths:  # find on path, the path order matters (as the candidates are less easy to control by end user)
+    # find on path, the path order matters (as the candidates are less easy to control by end user)
+    for pos, path in enumerate(paths):
+        path = six.ensure_text(path)
+        logging.debug(LazyPathDump(pos, path))
         for candidate, match in possible_specs(spec):
             found = check_path(candidate, path)
             if found is not None:
@@ -85,31 +88,25 @@ def get_paths():
         paths = []
     else:
         paths = [p for p in path.split(os.pathsep) if os.path.exists(p)]
-    logging.debug(LazyPathDump(paths))
     return paths
 
 
 class LazyPathDump(object):
-    def __init__(self, paths):
-        self.paths = paths
+    def __init__(self, pos, path):
+        self.pos = pos
+        self.path = path
 
     def __str__(self):
-        content = "PATH =>{}".format(os.linesep)
-        for i, p in enumerate(self.paths):
-            files = []
-            for file in Path(p).iterdir():
-                try:
-                    if file.is_dir():
-                        continue
-                except OSError:
-                    pass
-                files.append(file.name)
-            content += str(i)
+        content = "discover from PATH[{}]:{} with =>".format(self.pos, self.path)
+        for file_name in os.listdir(self.path):
+            try:
+                file_path = os.path.join(self.path, file_name)
+                if os.path.isdir(file_path) or not os.access(file_path, os.X_OK):
+                    continue
+            except OSError:
+                pass
             content += " "
-            content += str(p)
-            content += " with "
-            content += " ".join(files)
-            content += os.linesep
+            content += file_name
         return content
 
 
diff --git a/src/virtualenv/interpreters/discovery/py_info.py b/src/virtualenv/interpreters/discovery/py_info.py
index 2af2394..13e9b79 100644
--- a/src/virtualenv/interpreters/discovery/py_info.py
+++ b/src/virtualenv/interpreters/discovery/py_info.py
@@ -10,12 +10,9 @@ import json
 import logging
 import os
 import platform
-import subprocess
 import sys
 from collections import OrderedDict, namedtuple
 
-IS_WIN = sys.platform == "win32"
-
 VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"])
 
 
@@ -205,12 +202,14 @@ class PythonInfo(object):
 
     @classmethod
     def _load_for_exe(cls, exe):
+        from virtualenv.util.subprocess import subprocess, Popen
+
         path = "{}.py".format(os.path.splitext(__file__)[0])
         cmd = [exe, path]
         # noinspection DuplicatedCode
         # this is duplicated here because this file is executed on its own, so cannot be refactored otherwise
         try:
-            process = subprocess.Popen(
+            process = Popen(
                 cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE
             )
             out, err = process.communicate()