Merged in ludwigf/tox (pull request #114)

set VIRTUAL_ENV for test commands

12 files changed, 361 insertions, 73 deletions

diff --git a/CHANGELOG b/CHANGELOG
index 88babdc..9e69564 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,15 @@
+1.8.0.dev1
+-----------
+
+- new multi-dimensional configuration support.  Many thanks to
+  Alexander Schepanovski for the complete PR with docs.
+  And to Mike Bayer for filing an issue wrt to setting booleans.
+
+- fix issue148: remove "__PYVENV_LAUNCHER__" from os.environ when starting
+  subprocesses. Thanks Steven Myint.
+
+
+
 1.7.2
 -----------
 
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
index d10950b..ef8576d 100644
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -3,6 +3,7 @@ contributions:
 
 Krisztian Fekete
 Marc Abramowitz
+Aleaxner Schepanovski
 Sridhar Ratnakumar
 Barry Warsaw
 Chris Rose
diff --git a/doc/Makefile b/doc/Makefile
index 77b2083..d3393d6 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -37,7 +37,7 @@ clean:
 	-rm -rf $(BUILDDIR)/*
 
 install: clean html 
-	@rsync -avz $(BUILDDIR)/html/ testrun.org:/www/testrun.org/tox/latest
+	@rsync -avz $(BUILDDIR)/html/ testrun.org:/www/testrun.org/tox/dev
     #latexpdf
 	#@scp $(BUILDDIR)/latex/*.pdf testrun.org:www-tox/latest
 
diff --git a/doc/conf.py b/doc/conf.py
index ceb3397..0221f3c 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -48,7 +48,8 @@ copyright = u'2013, holger krekel and others'
 # built documents.
 #
 # The short X.Y version.
-release = version = "1.7.2"
+release = "1.8"
+version = "1.8.0.dev"
 # The full version, including alpha/beta/rc tags.
 
 # The language for content autogenerated by Sphinx. Refer to documentation
diff --git a/doc/config.txt b/doc/config.txt
index 9288ee2..8ce64db 100644
--- a/doc/config.txt
+++ b/doc/config.txt
@@ -382,6 +382,152 @@ You can put default values in one section and reference them in others to avoid
         {[base]deps}
 
 
+Generating environments, conditional settings
+---------------------------------------------
+
+.. versionadded:: 1.8
+
+Suppose you want to test your package against python2.6, python2.7 and against
+several versions of a dependency, say Django 1.5 and Django 1.6. You can
+accomplish that by writing down 2*2 = 4 ``[testenv:*]`` sections and then
+listing all of them in ``envlist``.
+
+However, a better approach looks like this::
+
+    [tox]
+    envlist = {py26,py27}-django{15,16}
+
+    [testenv]
+    basepython =
+        py26: python2.6
+        py27: python2.7
+    deps =
+        pytest
+        django15: Django>=1.5,<1.6
+        django16: Django>=1.6,<1.7
+        py26: unittest2
+    commands = py.test
+
+This uses two new facilities of tox-1.8:
+
+- generative envlist declarations where each envname
+  consists of environment parts or "factors"
+
+- "factor" specific settings
+
+Let's go through this step by step.
+
+
+Generative envlist
++++++++++++++++++++++++
+
+::
+
+    envlist = {py26,py27}-django{15,16}
+
+This is bash-style syntax and will create ``2*2=4`` environment names
+like this::
+
+    py26-django15
+    py26-django16
+    py27-django15
+    py27-django16
+
+You can still list environments explicitly along with generated ones::
+
+    envlist = {py26,py27}-django{15,16}, docs, flake
+
+.. note::
+
+    To help with understanding how the variants will produce section values,
+    you can ask tox to show their expansion with a new option::
+
+        $ tox -l
+        py26-django15
+        py26-django16
+        py27-django15
+        py27-django16
+        docs
+        flake
+
+
+Factors and factor-conditional settings
+++++++++++++++++++++++++++++++++++++++++
+
+Parts of an environment name delimited by hyphens are called factors and can
+be used to set values conditionally::
+
+    basepython =
+        py26: python2.6
+        py27: python2.7
+
+This conditional setting will lead to either ``python2.6`` or
+``python2.7`` used as base python, e.g. ``python2.6`` is selected if current
+environment contains ``py26`` factor.
+
+In list settings such as ``deps`` or ``commands`` you can freely intermix
+optional lines with unconditional ones::
+
+    deps =
+        pytest
+        django15: Django>=1.5,<1.6
+        django16: Django>=1.6,<1.7
+        py26: unittest2
+
+Reading it line by line:
+
+- ``pytest`` will be included unconditionally,
+- ``Django>=1.5,<1.6`` will be included for environments containing ``django15`` factor,
+- ``Django>=1.6,<1.7`` similarly depends on ``django16`` factor,
+- ``unittest`` will be loaded for Python 2.6 environments.
+
+.. note::
+
+    Tox provides good defaults for basepython setting, so the above
+    ini-file can be further reduced by omitting the ``basepython``
+    setting.
+
+
+Complex factor conditions
++++++++++++++++++++++++++
+
+Sometimes you need to specify same line for several factors or create a special
+case for a combination of factors. Here is how you do it::
+
+    [tox]
+    envlist = py{26,27,33}-django{15,16}-{sqlite,mysql}
+
+    [testenv]
+    deps =
+        py33-mysql: PyMySQL     ; use if both py33 and mysql are in an env name
+        py26,py27: urllib3      ; use if any of py26 or py27 are in an env name
+        py{26,27}-sqlite: mock  ; mocking sqlite in python 2.x
+
+Take a look at first ``deps`` line. It shows how you can special case something
+for a combination of factors, you just join combining factors with a hyphen.
+This particular line states that ``PyMySQL`` will be loaded for python 3.3,
+mysql environments, e.g. ``py33-django15-mysql`` and ``py33-django16-mysql``.
+
+The second line shows how you use same line for several factors - by listing
+them delimited by commas. It's possible to list not only simple factors, but
+also their combinations like ``py26-sqlite,py27-sqlite``.
+
+Finally, factor expressions are expanded the same way as envlist, so last
+example could be rewritten as ``py{26,27}-sqlite``.
+
+.. note::
+
+    Factors don't do substring matching against env name, instead every
+    hyphenated expression is split by ``-`` and if ALL the factors in an
+    expression are also factors of an env then that condition is considered
+    hold.
+
+    For example, environment ``py26-mysql``:
+
+    - could be matched with expressions ``py26``, ``py26-mysql``,
+      ``mysql-py26``,
+    - but not with ``py2`` or ``py26-sql``.
+
 
 Other Rules and notes
 =====================
diff --git a/doc/example/basic.txt b/doc/example/basic.txt
index 6b115c8..562e2bf 100644
--- a/doc/example/basic.txt
+++ b/doc/example/basic.txt
@@ -41,6 +41,7 @@ Available "default" test environments names are::
     py34
     jython
     pypy
+    pypy3
 
 However, you can also create your own test environment names,
 see some of the examples in :doc:`examples <../examples>`.
diff --git a/setup.py b/setup.py
index 8e2f8f0..a9ba348 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ def main():
         description='virtualenv-based automation of test activities',
         long_description=open("README.rst").read(),
         url='http://tox.testrun.org/',
-        version='1.7.2',
+        version='1.8.0.dev2',
         license='http://opensource.org/licenses/MIT',
         platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'],
         author='holger krekel',
diff --git a/tests/test_config.py b/tests/test_config.py
index bc5a683..efbbc80 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -832,6 +832,71 @@ class TestConfigTestEnv:
         assert conf.changedir.basename == 'testing'
         assert conf.changedir.dirpath().realpath() == tmpdir.realpath()
 
+    def test_factors(self, newconfig):
+        inisource="""
+            [tox]
+            envlist = a-x,b
+
+            [testenv]
+            deps=
+                dep-all
+                a: dep-a
+                b: dep-b
+                x: dep-x
+        """
+        conf = newconfig([], inisource)
+        configs = conf.envconfigs
+        assert [dep.name for dep in configs['a-x'].deps] == \
+            ["dep-all", "dep-a", "dep-x"]
+        assert [dep.name for dep in configs['b'].deps] == ["dep-all", "dep-b"]
+
+    def test_factor_ops(self, newconfig):
+        inisource="""
+            [tox]
+            envlist = {a,b}-{x,y}
+
+            [testenv]
+            deps=
+                a,b: dep-a-or-b
+                a-x: dep-a-and-x
+                {a,b}-y: dep-ab-and-y
+        """
+        configs = newconfig([], inisource).envconfigs
+        get_deps = lambda env: [dep.name for dep in configs[env].deps]
+        assert get_deps("a-x") == ["dep-a-or-b", "dep-a-and-x"]
+        assert get_deps("a-y") == ["dep-a-or-b", "dep-ab-and-y"]
+        assert get_deps("b-x") == ["dep-a-or-b"]
+        assert get_deps("b-y") == ["dep-a-or-b", "dep-ab-and-y"]
+
+    def test_default_factors(self, newconfig):
+        inisource="""
+            [tox]
+            envlist = py{26,27,33,34}-dep
+
+            [testenv]
+            deps=
+                dep: dep
+        """
+        conf = newconfig([], inisource)
+        configs = conf.envconfigs
+        for name, config in configs.items():
+            assert config.basepython == 'python%s.%s' % (name[2], name[3])
+
+    @pytest.mark.issue188
+    def test_factors_in_boolean(self, newconfig):
+        inisource="""
+            [tox]
+            envlist = py{27,33}
+
+            [testenv]
+            recreate =
+                py27: True
+        """
+        configs = newconfig([], inisource).envconfigs
+        assert configs["py27"].recreate
+        assert not configs["py33"].recreate
+
+
 class TestGlobalOptions:
     def test_notest(self, newconfig):
         config = newconfig([], "")
@@ -935,6 +1000,23 @@ class TestGlobalOptions:
                 bp = "python%s.%s" %(name[2], name[3])
                 assert env.basepython == bp
 
+    def test_envlist_expansion(self, newconfig):
+        inisource = """
+            [tox]
+            envlist = py{26,27},docs
+        """
+        config = newconfig([], inisource)
+        assert config.envlist == ["py26", "py27", "docs"]
+
+    def test_envlist_cross_product(self, newconfig):
+        inisource = """
+            [tox]
+            envlist = py{26,27}-dep{1,2}
+        """
+        config = newconfig([], inisource)
+        assert config.envlist == \
+            ["py26-dep1", "py26-dep2", "py27-dep1", "py27-dep2"]
+
     def test_minversion(self, tmpdir, newconfig, monkeypatch):
         inisource = """
             [tox]
diff --git a/tests/test_venv.py b/tests/test_venv.py
index 6482e09..00e5f8b 100644
--- a/tests/test_venv.py
+++ b/tests/test_venv.py
@@ -485,9 +485,11 @@ class TestVenvTest:
         py.test.raises(ZeroDivisionError, "venv._pcall([1,2,3])")
         monkeypatch.setenv("PIP_RESPECT_VIRTUALENV", "1")
         monkeypatch.setenv("PIP_REQUIRE_VIRTUALENV", "1")
+        monkeypatch.setenv("__PYVENV_LAUNCHER__", "1")
         py.test.raises(ZeroDivisionError, "venv.run_install_command(['qwe'])")
         assert 'PIP_RESPECT_VIRTUALENV' not in os.environ
         assert 'PIP_REQUIRE_VIRTUALENV' not in os.environ
+        assert '__PYVENV_LAUNCHER__' not in os.environ
 
 def test_setenv_added_to_pcall(tmpdir, mocksession, newconfig):
     pkg = tmpdir.ensure("package.tar.gz")
@@ -552,8 +554,6 @@ def test_run_install_command(newmocksession):
     assert 'install' in l[0].args
     env = l[0].env
     assert env is not None
-    assert 'PYTHONIOENCODING' in env
-    assert env['PYTHONIOENCODING'] == 'utf_8'
 
 def test_run_custom_install_command(newmocksession):
     mocksession = newmocksession([], """
diff --git a/tox/__init__.py b/tox/__init__.py
index 530f3c4..775ed11 100644
--- a/tox/__init__.py
+++ b/tox/__init__.py
@@ -1,5 +1,5 @@
 #
-__version__ = '1.7.2'
+__version__ = '1.8.0.dev2'
 
 class exception:
     class Error(Exception):
diff --git a/tox/_config.py b/tox/_config.py
index 1495bd8..07f2365 100644
--- a/tox/_config.py
+++ b/tox/_config.py
@@ -6,6 +6,7 @@ import re
 import shlex
 import string
 import pkg_resources
+import itertools
 
 from tox.interpreters import Interpreters
 
@@ -15,13 +16,10 @@ import tox
 
 iswin32 = sys.platform == "win32"
 
-defaultenvs = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3'}
-for _name in "py,py24,py25,py26,py27,py30,py31,py32,py33,py34".split(","):
-    if _name == "py":
-        basepython = sys.executable
-    else:
-        basepython = "python" + ".".join(_name[2:4])
-    defaultenvs[_name] = basepython
+default_factors = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3',
+                   'py': sys.executable}
+for version in '24,25,26,27,30,31,32,33,34'.split(','):
+    default_factors['py' + version] = 'python%s.%s' % tuple(version)
 
 def parseconfig(args=None, pkg=None):
     if args is None:
@@ -280,22 +278,19 @@ class parseini:
         config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None)
         config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}")
         config.logdir = config.toxworkdir.join("log")
-        for sectionwrapper in self._cfg:
-            section = sectionwrapper.name
-            if section.startswith(testenvprefix):
-                name = section[len(testenvprefix):]
-                envconfig = self._makeenvconfig(name, section, reader._subs,
-                    config)
-                config.envconfigs[name] = envconfig
-        if not config.envconfigs:
-            config.envconfigs['python'] = \
-                self._makeenvconfig("python", "_xz_9", reader._subs, config)
-        config.envlist = self._getenvlist(reader, toxsection)
-        for name in config.envlist:
-            if name not in config.envconfigs:
-                if name in defaultenvs:
-                    config.envconfigs[name] = \
-                self._makeenvconfig(name, "_xz_9", reader._subs, config)
+
+        config.envlist, all_envs = self._getenvdata(reader, toxsection)
+
+        # configure testenvs
+        known_factors = self._list_section_factors("testenv")
+        known_factors.update(default_factors)
+        known_factors.add("python")
+        for name in all_envs:
+            section = testenvprefix + name
+            factors = set(name.split('-'))
+            if section in self._cfg or factors <= known_factors:
+                config.envconfigs[name] = \
+                    self._makeenvconfig(name, section, reader._subs, config)
 
         all_develop = all(name in config.envconfigs
                           and config.envconfigs[name].develop
@@ -303,10 +298,20 @@ class parseini:
 
         config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop)
 
+    def _list_section_factors(self, section):
+        factors = set()
+        if section in self._cfg:
+            for _, value in self._cfg[section].items():
+                exprs = re.findall(r'^([\w{},-]+)\:\s+', value, re.M)
+                factors.update(*mapcat(_split_factor_expr, exprs))
+        return factors
+
     def _makeenvconfig(self, name, section, subs, config):
         vc = VenvConfig(envname=name)
         vc.config = config
-        reader = IniReader(self._cfg, fallbacksections=["testenv"])
+        factors = set(name.split('-'))
+        reader = IniReader(self._cfg, fallbacksections=["testenv"],
+            factors=factors)
         reader.addsubstitutions(**subs)
         vc.develop = not config.option.installpkg and \
                reader.getbool(section, "usedevelop", config.option.develop)
@@ -315,10 +320,8 @@ class parseini:
         if reader.getdefault(section, "python", None):
             raise tox.exception.ConfigError(
                 "'python=' key was renamed to 'basepython='")
-        if name in defaultenvs:
-            bp = defaultenvs[name]
-        else:
-            bp = sys.executable
+        bp = next((default_factors[f] for f in factors if f in default_factors),
+            sys.executable)
         vc.basepython = reader.getdefault(section, "basepython", bp)
         vc._basepython_info = config.interpreters.get_info(vc.basepython)
         reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname,
@@ -386,20 +389,25 @@ class parseini:
              "'install_command' must contain '{packages}' substitution")
         return vc
 
-    def _getenvlist(self, reader, toxsection):
-        env = self.config.option.env
-        if not env:
-            env = os.environ.get("TOXENV", None)
-            if not env:
-                envlist = reader.getlist(toxsection, "envlist", sep=",")
-                if not envlist:
-                    envlist = self.config.envconfigs.keys()
-                return envlist
-        envlist = _split_env(env)
-        if "ALL" in envlist:
-            envlist = list(self.config.envconfigs)
-            envlist.sort()
-        return envlist
+    def _getenvdata(self, reader, toxsection):
+        envstr = self.config.option.env                                \
+            or os.environ.get("TOXENV")                                \
+            or reader.getdefault(toxsection, "envlist", replace=False) \
+            or []
+        envlist = _split_env(envstr)
+
+        # collect section envs
+        all_envs = set(envlist) - set(["ALL"])
+        for section in self._cfg:
+            if section.name.startswith(testenvprefix):
+                all_envs.add(section.name[len(testenvprefix):])
+        if not all_envs:
+            all_envs.add("python")
+
+        if not envlist or "ALL" in envlist:
+            envlist = sorted(all_envs)
+
+        return envlist, all_envs
 
     def _replace_forced_dep(self, name, config):
         """
@@ -427,17 +435,32 @@ class parseini:
         dep2_name = pkg_resources.Requirement.parse(dep2).project_name
         return dep1_name == dep2_name
 
+
 def _split_env(env):
     """if handed a list, action="append" was used for -e """
-    envlist = []
     if not isinstance(env, list):
         env = [env]
-    for to_split in env:
-        for single_env in to_split.split(","):
-            # "remove True or", if not allowing multiple same runs, update tests
-            if True or single_env not in envlist:
-                envlist.append(single_env)
-    return envlist
+    return mapcat(_expand_envstr, env)
+
+def _split_factor_expr(expr):
+    partial_envs = _expand_envstr(expr)
+    return [set(e.split('-')) for e in partial_envs]
+
+def _expand_envstr(envstr):
+    # split by commas not in groups
+    tokens = re.split(r'(\{[^}]+\})|,', envstr)
+    envlist = [''.join(g).strip()
+               for k, g in itertools.groupby(tokens, key=bool) if k]
+
+    def expand(env):
+        tokens = re.split(r'\{([^}]+)\}', env)
+        parts = [token.split(',') for token in tokens]
+        return [''.join(variant) for variant in itertools.product(*parts)]
+
+    return mapcat(expand, envlist)
+
+def mapcat(f, seq):
+    return list(itertools.chain.from_iterable(map(f, seq)))
 
 class DepConfig:
     def __init__(self, name, indexserver=None):
@@ -468,9 +491,10 @@ RE_ITEM_REF = re.compile(
 
 
 class IniReader:
-    def __init__(self, cfgparser, fallbacksections=None):
+    def __init__(self, cfgparser, fallbacksections=None, factors=()):
         self._cfg = cfgparser
         self.fallbacksections = fallbacksections or []
+        self.factors = factors
         self._subs = {}
         self._subststack = []
 
@@ -572,9 +596,12 @@ class IniReader:
 
     def getbool(self, section, name, default=None):
         s = self.getdefault(section, name, default)
+        if not s:
+            s = default
         if s is None:
             raise KeyError("no config value [%s] %s found" % (
                 section, name))
+
         if not isinstance(s, bool):
             if s.lower() == "true":
                 s = True
@@ -586,18 +613,19 @@ class IniReader:
         return s
 
     def getdefault(self, section, name, default=None, replace=True):
-        try:
-            x = self._cfg[section][name]
-        except KeyError:
-            for fallbacksection in self.fallbacksections:
-                try:
-                    x = self._cfg[fallbacksection][name]
-                except KeyError:
-                    pass
-                else:
-                    break
-            else:
-                x = default
+        x = None
+        for s in [section] + self.fallbacksections:
+            try:
+                x = self._cfg[s][name]
+                break
+            except KeyError:
+                continue
+
+        if x is None:
+            x = default
+        else:
+            x = self._apply_factors(x)
+
         if replace and x and hasattr(x, 'replace'):
             self._subststack.append((section, name))
             try:
@@ -607,6 +635,19 @@ class IniReader:
         #print "getdefault", section, name, "returned", repr(x)
         return x
 
+    def _apply_factors(self, s):
+        def factor_line(line):
+            m = re.search(r'^([\w{},-]+)\:\s+(.+)', line)
+            if not m:
+                return line
+
+            expr, line = m.groups()
+            if any(fs <= self.factors for fs in _split_factor_expr(expr)):
+                return line
+
+        lines = s.strip().splitlines()
+        return '\n'.join(filter(None, map(factor_line, lines)))
+
     def _replace_env(self, match):
         match_value = match.group('substitution_value')
         if not match_value:
diff --git a/tox/_venv.py b/tox/_venv.py
index df37eac..0c56cc0 100644
--- a/tox/_venv.py
+++ b/tox/_venv.py
@@ -1,5 +1,6 @@
 from __future__ import with_statement
 import sys, os
+import codecs
 import py
 import tox
 from tox._config import DepConfig
@@ -271,16 +272,19 @@ class VirtualEnv(object):
         if '{opts}' in argv:
             i = argv.index('{opts}')
             argv[i:i+1] = list(options)
-        for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV'):
+        for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV',
+                  '__PYVENV_LAUNCHER__'):
             try:
                 del os.environ[x]
             except KeyError:
                 pass
-        env = dict(PYTHONIOENCODING='utf_8')
-        if extraenv is not None:
-            env.update(extraenv)
+        old_stdout = sys.stdout
+        sys.stdout = codecs.getwriter('utf8')(sys.stdout)
+        if extraenv is None:
+            extraenv = {}
         self._pcall(argv, cwd=self.envconfig.config.toxinidir,
-                    extraenv=env, action=action)
+                    extraenv=extraenv, action=action)
+        sys.stdout = old_stdout
 
     def _install(self, deps, extraopts=None, action=None):
         if not deps: