Merge branch 'master' into envsubs-independence

11 files changed, 258 insertions, 50 deletions

diff --git a/.gitignore b/.gitignore
index cfc664c8..c6ff41cd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@
 *.orig
 *.rej
 *~
+__pycache__
 
 build
 dist
@@ -14,3 +15,7 @@ doc/_build/
 tox.egg-info
 .tox
 .cache
+
+.idea
+.eggs/
+py27/
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index 1a163474..84ccf207 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -46,3 +46,6 @@ Paweł Adamczak
 Oliver Bestwalter
 Selim Belhaouane
 Nick Douma
+Cyril Roelandt
+Bartolome Sanchez Salado
+Laszlo Vasko
diff --git a/doc/config.txt b/doc/config.txt
index 260a1ac5..8880123c 100644
--- a/doc/config.txt
+++ b/doc/config.txt
@@ -214,11 +214,16 @@ Complete list of settings that you can put into ``testenv*`` sections:
    You can use ``*`` and ``?`` to match multiple environment variables with
    one name.
 
-   Note that the ``PATH``, ``LANG``, ``LANGUAGE`` and ``PIP_INDEX_URL``
-   variables are unconditionally passed down and on Windows ``SYSTEMROOT``,
-   ``PATHEXT``, ``TEMP`` and ``TMP`` will be passed down as well whereas on
-   unix ``TMPDIR`` will be passed down.  You can override these variables
-   with the ``setenv`` option.
+   Some variables are always passed through to ensure the basic functionality
+   of standard library functions or tooling like pip:
+
+   * passed through on all platforms: ``PATH``, ``LANG``, ``LANGUAGE``,
+     ``LD_LIBRARY_PATH``, ``PIP_INDEX_URL``
+   * Windows: ``SYSTEMDRIVE``, ``SYSTEMROOT``, ``PATHEXT``, ``TEMP``, ``TMP``
+      ``NUMBER_OF_PROCESSORS``, ``USERPROFILE``, ``MSYSTEM``
+   * Others (e.g. UNIX, macOS): ``TMPDIR``
+
+   You can override these variables with the ``setenv`` option.
 
    If defined the ``TOX_TESTENV_PASSENV`` environment variable (in the tox
    invocation environment) can define additional space-separated variable
@@ -430,6 +435,14 @@ then the value will be retrieved as ``os.environ['KEY']``
 and replace with and empty string if the environment variable does not
 exist.
 
+Substitutions can also be nested. In that case they are expanded starting
+from the innermost expression::
+
+    {env:KEY:{env:DEFAULT_OF_KEY}}
+
+the above example is roughly equivalent to
+``os.environ.get('KEY', os.environ['DEFAULT_OF_KEY'])``
+
 .. _`command positional substitution`:
 .. _`positional substitution`:
 
diff --git a/tests/test_config.py b/tests/test_config.py
index 5e4889da..743b08ed 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -334,6 +334,16 @@ class TestIniParserAgainstCommandsKey:
         assert envconfig.commands == [["ls", "testvalue"]]
         assert envconfig.setenv["TEST"] == "testvalue"
 
+    def test_command_env_substitution_global(self, newconfig):
+        """Ensure referenced {env:key:default} values are substituted correctly."""
+        config = newconfig("""
+            [testenv]
+            setenv = FOO = bar
+            commands = echo {env:FOO}
+        """)
+        envconfig = config.envconfigs['python']
+        assert envconfig.commands == [["echo", "bar"]]
+
 
 class TestIniParser:
     def test_getstring_single(self, tmpdir, newconfig):
@@ -866,6 +876,7 @@ class TestConfigTestEnv:
             assert "TMP" in envconfig.passenv
             assert "NUMBER_OF_PROCESSORS" in envconfig.passenv
             assert "USERPROFILE" in envconfig.passenv
+            assert "MSYSTEM" in envconfig.passenv
         else:
             assert "TMPDIR" in envconfig.passenv
         assert "PATH" in envconfig.passenv
@@ -1098,7 +1109,7 @@ class TestConfigTestEnv:
         ]
 
     @pytest.mark.xfail(raises=AssertionError, reason="issue #301")
-    def test_substitution_env_defaults_issue301(tmpdir, newconfig, monkeypatch):
+    def test_substitution_nested_env_defaults_issue301(tmpdir, newconfig, monkeypatch):
         monkeypatch.setenv("IGNORE_STATIC_DEFAULT", "env")
         monkeypatch.setenv("IGNORE_DYNAMIC_DEFAULT", "env")
         config = newconfig("""
@@ -2179,6 +2190,20 @@ class TestCmdInvocation:
             r'*deps*dep1, dep2==5.0*',
         ])
 
+    @pytest.mark.xfail(reason='Upstream bug. See #203')
+    def test_colon_symbol_in_directory_name(self, cmd, initproj):
+        initproj('colon:_symbol_in_dir_name', filedefs={
+            'tox.ini': '''
+            [tox]
+            envlist = py27
+
+            [testenv]
+            commands = pip --version
+            ''',
+        })
+        result = cmd.run("tox")
+        assert result.ret == 0
+
 
 @pytest.mark.parametrize("cmdline,envlist", [
     ("-e py26", ['py26']),
diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py
index 49b2b0c0..76551008 100644
--- a/tests/test_quickstart.py
+++ b/tests/test_quickstart.py
@@ -379,6 +379,106 @@ deps =
         result = read_tox('tox-generated.ini')
         assert(result == expected_tox_ini)
 
+    def test_quickstart_main_tox_ini_location_can_be_overridden(
+            self,
+            tmpdir,
+            monkeypatch):
+        monkeypatch.setattr(
+            tox._quickstart, 'term_input',
+            self.get_mock_term_input(
+                [
+                    '1',          # py27 and py33
+                    'py.test',    # command to run tests
+                    '',            # test dependencies
+                ]
+            )
+        )
+
+        root_dir = tmpdir.mkdir('alt-root')
+        tox_ini_path = root_dir.join('tox.ini')
+
+        tox._quickstart.main(argv=['tox-quickstart', root_dir.basename])
+
+        assert tox_ini_path.isfile()
+
+        expected_tox_ini = """
+# Tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py27
+
+[testenv]
+commands = py.test
+deps =
+    pytest
+""".lstrip()
+        result = read_tox(fname=tox_ini_path.strpath)
+        assert(result == expected_tox_ini)
+
+    def test_quickstart_main_custom_tox_ini_location_with_existing_tox_ini(
+            self,
+            tmpdir,
+            monkeypatch):
+        monkeypatch.setattr(
+            tox._quickstart, 'term_input',
+            self.get_mock_term_input(
+                [
+                    '1',          # py27 and py33
+                    'py.test',    # command to run tests
+                    '',            # test dependencies
+                    '',           # tox.ini already exists; overwrite?
+                ]
+            )
+        )
+
+        root_dir = tmpdir.mkdir('alt-root')
+        tox_ini_path = root_dir.join('tox.ini')
+        tox_ini_path.write('foo\nbar\n')
+
+        tox._quickstart.main(argv=['tox-quickstart', root_dir.basename])
+        tox_ini_path = root_dir.join('tox-generated.ini')
+
+        assert tox_ini_path.isfile()
+
+        expected_tox_ini = """
+# Tox (https://tox.readthedocs.io/) is a tool for running tests
+# in multiple virtualenvs. This configuration file will run the
+# test suite on all supported python versions. To use it, "pip install tox"
+# and then run "tox" from this directory.
+
+[tox]
+envlist = py27
+
+[testenv]
+commands = py.test
+deps =
+    pytest
+""".lstrip()
+        result = read_tox(fname=tox_ini_path.strpath)
+        assert(result == expected_tox_ini)
+
+    def test_quickstart_main_custom_nonexistent_tox_ini_location(
+            self,
+            tmpdir,
+            monkeypatch):
+        monkeypatch.setattr(
+            tox._quickstart, 'term_input',
+            self.get_mock_term_input(
+                [
+                    '1',          # py27 and py33
+                    'py.test',    # command to run tests
+                    '',           # test dependencies
+                ]
+            )
+        )
+
+        root_dir = tmpdir.join('nonexistent-root')
+
+        assert tox._quickstart.main(argv=['tox-quickstart', root_dir.basename]) == 2
+
 
 class TestToxQuickstart(object):
     def test_pytest(self):
diff --git a/tox.ini b/tox.ini
index c7469810..07c0ffb0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,49 +1,50 @@
 [tox]
-envlist=py27,py26,py34,py33,py35,py36,pypy,flakes,py26-bare
+envlist = py27,py26,py34,py33,py35,py36,pypy,flakes,py26-bare
+minversion = 2.7.0
 
 [testenv:X]
-commands=echo {posargs}
+commands = echo {posargs}
+description = print the positional arguments passed in with echo
 
 [testenv]
-commands= pytest --timeout=180 {posargs:tests}
-
-deps=pytest>=3.0.0
-    pytest-timeout
+commands = pytest --timeout=180 {posargs:tests}
+description = run the unit tests with pytest under the current Python env
+deps = pytest >= 3.0.0
+       pytest-timeout
 
 [testenv:py26-bare]
 deps =
 commands = tox -h
 install_command = pip install {opts} {packages}
 list_dependencies_command = pip freeze
+description = invoke the tox help message under Python 2.6
 
 [testenv:py26]
 install_command = pip install {opts} {packages}
 list_dependencies_command = pip freeze
 
-
 [testenv:docs]
-basepython=python
-changedir=doc
-deps=sphinx
-     {[testenv]deps}
-commands=
-    pytest -v check_sphinx.py {posargs}
+basepython = python
+changedir = doc
+deps = sphinx
+       {[testenv]deps}
+commands = pytest -v check_sphinx.py {posargs}
+description = invoke sphinx-build and try to build the HTML page (also check link validity)
 
 [testenv:flakes]
-deps = pytest-flakes>=0.2
+deps = pytest-flakes >= 0.2
        pytest-pep8
-
-commands =
-    pytest --flakes -m flakes tox tests
-    pytest --pep8 -m pep8 tox tests
+description = run static analysis and style check using flakes and pep-8
+commands = pytest --flakes -m flakes tox tests
+           pytest --pep8 -m pep8 tox tests
 
 [testenv:dev]
 # required to make looponfail reload on every source code change
 usedevelop = True
 
-deps =
-      pytest-xdist>=1.11
-commands = {posargs:pytest -s -x -f -v}
+deps = pytest-xdist >= 1.11
+commands = {posargs:py.test -s -x -f -v}
+description = DEV enviroment, if no posarg is specified: run pytest
 
 [pytest]
 rsyncdirs = tests tox
@@ -57,6 +58,5 @@ pep8maxlinelength = 99
 # W503 - line break before binary operator
 # E402 - module level import not at top of file
 # E731 - do not assign a lambda expression, use a def
-pep8ignore =
-           *.py W503 E402 E731
+pep8ignore = *.py W503 E402 E731
 flakes-ignore = ImportStarUsage
diff --git a/tox/_quickstart.py b/tox/_quickstart.py
index 37c48ddc..bc283a8a 100644
--- a/tox/_quickstart.py
+++ b/tox/_quickstart.py
@@ -40,6 +40,7 @@
     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 """
 
+import argparse
 import sys
 from os import path
 from codecs import open
@@ -224,19 +225,24 @@ def generate(d, overwrite=True, silent=False):
 
     def write_file(fpath, mode, content):
         print('Creating file %s.' % fpath)
-        f = open(fpath, mode, encoding='utf-8')
         try:
-            f.write(content)
-        finally:
-            f.close()
+            with open(fpath, mode, encoding='utf-8') as f:
+                f.write(content)
+        except IOError:
+            print('Error writing file.')
+            raise
 
     sys.stdout.write('\n')
 
-    fpath = 'tox.ini'
+    fpath = path.join(d.get('path', ''), 'tox.ini')
 
     if path.isfile(fpath) and not overwrite:
         print('File %s already exists.' % fpath)
-        do_prompt(d, 'fpath', 'Alternative path to write tox.ini contents to', 'tox-generated.ini')
+        do_prompt(
+            d,
+            'fpath',
+            'Alternative path to write tox.ini contents to',
+            path.join(d.get('path', ''), 'tox-generated.ini'))
         fpath = d['fpath']
 
     write_file(fpath, 'w', conf_text)
@@ -251,14 +257,25 @@ Execute `tox` to test your project.
 ''')
 
 
+def parse_args(argv):
+    parser = argparse.ArgumentParser(
+        description='Command-line script to quickly setup tox.ini for a Python project.'
+    )
+    parser.add_argument(
+        'root', type=str, nargs='?', default='.',
+        help='Custom root directory to write tox.ini to. Defaults to current directory.'
+    )
+    parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)
+
+    args = argv[1:]
+    return parser.parse_args(args)
+
+
 def main(argv=sys.argv):
-    d = {}
+    args = parse_args(argv)
 
-    if len(argv) > 3:
-        print('Usage: tox-quickstart [root]')
-        sys.exit(1)
-    elif len(argv) == 2:
-        d['path'] = argv[1]
+    d = {}
+    d['path'] = args.root
 
     try:
         ask_user(d)
@@ -268,8 +285,13 @@ def main(argv=sys.argv):
         return
 
     d = process_input(d)
-    generate(d, overwrite=False)
+    try:
+        generate(d, overwrite=False)
+    except Exception:
+        return 2
+
+    return 0
 
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())
diff --git a/tox/config.py b/tox/config.py
index 67501b9d..e5e616b3 100755
--- a/tox/config.py
+++ b/tox/config.py
@@ -492,7 +492,8 @@ def tox_addoption(parser):
             # for `multiprocessing.cpu_count()` on Windows
             # (prior to Python 3.4).
             passenv.add("NUMBER_OF_PROCESSORS")
-            passenv.add("USERPROFILE")  # needed for `os.path.expanduser()`.
+            passenv.add("USERPROFILE")  # needed for `os.path.expanduser()`
+            passenv.add("MSYSTEM")      # fixes #429
         else:
             passenv.add("TMPDIR")
         for spec in value:
@@ -835,7 +836,7 @@ class parseini:
                 res = env_attr.postprocess(testenv_config=vc, value=res)
             setattr(vc, env_attr.name, res)
 
-            if atype == "path":
+            if atype in ("path", "string"):
                 reader.addsubstitutions(**{env_attr.name: res})
 
         return vc
@@ -1059,8 +1060,20 @@ class Replacer:
         self.reader = reader
         self.crossonly = crossonly
 
-    def do_replace(self, x):
-        return self.RE_ITEM_REF.sub(self._replace_match, x)
+    def do_replace(self, value):
+        '''
+        Recursively expand substitutions starting from the innermost expression
+        '''
+        def substitute_once(x):
+            return self.RE_ITEM_REF.sub(self._replace_match, x)
+
+        expanded = substitute_once(value)
+
+        while expanded != value:  # substitution found
+            value = expanded
+            expanded = substitute_once(value)
+
+        return expanded
 
     def _replace_match(self, match):
         g = match.groupdict()
diff --git a/tox/hookspecs.py b/tox/hookspecs.py
index f4ca7422..0e88b2be 100644
--- a/tox/hookspecs.py
+++ b/tox/hookspecs.py
@@ -88,6 +88,15 @@ def tox_runtest_pre(venv):
     """
 
 
+@hookspec(firstresult=True)
+def tox_runtest(venv, redirect):
+    """ [experimental] run the tests for this venv.
+
+    .. note:: This hook uses ``firstresult=True`` (see pluggy_) -- hooks
+        implementing this will be run until one returns non-``None``.
+    """
+
+
 @hookspec
 def tox_runtest_post(venv):
     """ [experimental] perform arbitrary action after running tests for this venv.
diff --git a/tox/session.py b/tox/session.py
index fd726ac5..626d4a7f 100644
--- a/tox/session.py
+++ b/tox/session.py
@@ -449,8 +449,19 @@ class Session:
             envlog = self.resultlog.get_envlog(venv.name)
             try:
                 status = venv.update(action=action)
-            except tox.exception.InvocationError:
-                status = sys.exc_info()[1]
+            except FileNotFoundError as e:
+                status = (
+                    "Error creating virtualenv. "
+                    "Note that spaces in path are not supported by virtualenv. "
+                    "Error details: %r" % e
+                )
+            except tox.exception.InvocationError as e:
+                status = (
+                    "Error creating virtualenv. "
+                    "Note that some special characters (such as ':' and unicode symbols) "
+                    "in path are not supported by virtualenv. "
+                    "Error details: %r" % e
+                )
             if status:
                 commandlog = envlog.get_commandlog("setup")
                 commandlog.add_command(["setup virtualenv"], str(status), 1)
@@ -571,7 +582,7 @@ class Session:
             if venv.status:
                 return
             self.hook.tox_runtest_pre(venv=venv)
-            venv.test(redirect=redirect)
+            self.hook.tox_runtest(venv=venv, redirect=redirect)
             self.hook.tox_runtest_post(venv=venv)
         else:
             venv.status = "skipped tests"
diff --git a/tox/venv.py b/tox/venv.py
index 4e2993a4..509a9b11 100755
--- a/tox/venv.py
+++ b/tox/venv.py
@@ -455,3 +455,10 @@ def tox_testenv_install_deps(venv, action):
         venv._install(deps, action=action)
     # Return non-None to indicate the plugin has completed
     return True
+
+
+@hookimpl
+def tox_runtest(venv, redirect):
+    venv.test(redirect=redirect)
+    # Return non-None to indicate the plugin has completed
+    return True