diff options
author | holger krekel <holger@merlinux.eu> | 2014-09-23 16:08:54 +0200 |
---|---|---|
committer | holger krekel <holger@merlinux.eu> | 2014-09-23 16:08:54 +0200 |
commit | dbb4723075fe76c0b61af8c0f733e0d75e555a60 (patch) | |
tree | 127bca45ab7c7f4e86616c3adb38bdb97dc56572 | |
parent | 7c8828adf6819dda1ead6c7ee419714f2fc42b8c (diff) | |
parent | d5d42e21988896981f2028b1d986f13c3b5cbeb8 (diff) | |
download | tox-dbb4723075fe76c0b61af8c0f733e0d75e555a60.tar.gz |
Merged in ludwigf/tox (pull request #114)
set VIRTUAL_ENV for test commands
-rw-r--r-- | CHANGELOG | 12 | ||||
-rw-r--r-- | CONTRIBUTORS | 1 | ||||
-rw-r--r-- | doc/Makefile | 2 | ||||
-rw-r--r-- | doc/conf.py | 3 | ||||
-rw-r--r-- | doc/config.txt | 146 | ||||
-rw-r--r-- | doc/example/basic.txt | 1 | ||||
-rw-r--r-- | setup.py | 2 | ||||
-rw-r--r-- | tests/test_config.py | 82 | ||||
-rw-r--r-- | tests/test_venv.py | 4 | ||||
-rw-r--r-- | tox/__init__.py | 2 | ||||
-rw-r--r-- | tox/_config.py | 165 | ||||
-rw-r--r-- | tox/_venv.py | 14 |
12 files changed, 361 insertions, 73 deletions
@@ -1,3 +1,15 @@ +1.8.0.dev1 +----------- + +- new multi-dimensional configuration support. Many thanks to + Alexander Schepanovski for the complete PR with docs. + And to Mike Bayer for filing an issue wrt to setting booleans. + +- fix issue148: remove "__PYVENV_LAUNCHER__" from os.environ when starting + subprocesses. Thanks Steven Myint. + + + 1.7.2 ----------- diff --git a/CONTRIBUTORS b/CONTRIBUTORS index d10950b..ef8576d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,6 +3,7 @@ contributions: Krisztian Fekete Marc Abramowitz +Aleaxner Schepanovski Sridhar Ratnakumar Barry Warsaw Chris Rose diff --git a/doc/Makefile b/doc/Makefile index 77b2083..d3393d6 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -37,7 +37,7 @@ clean: -rm -rf $(BUILDDIR)/* install: clean html - @rsync -avz $(BUILDDIR)/html/ testrun.org:/www/testrun.org/tox/latest + @rsync -avz $(BUILDDIR)/html/ testrun.org:/www/testrun.org/tox/dev #latexpdf #@scp $(BUILDDIR)/latex/*.pdf testrun.org:www-tox/latest diff --git a/doc/conf.py b/doc/conf.py index ceb3397..0221f3c 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -48,7 +48,8 @@ copyright = u'2013, holger krekel and others' # built documents. # # The short X.Y version. -release = version = "1.7.2" +release = "1.8" +version = "1.8.0.dev" # The full version, including alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation diff --git a/doc/config.txt b/doc/config.txt index 9288ee2..8ce64db 100644 --- a/doc/config.txt +++ b/doc/config.txt @@ -382,6 +382,152 @@ You can put default values in one section and reference them in others to avoid {[base]deps} +Generating environments, conditional settings +--------------------------------------------- + +.. versionadded:: 1.8 + +Suppose you want to test your package against python2.6, python2.7 and against +several versions of a dependency, say Django 1.5 and Django 1.6. You can +accomplish that by writing down 2*2 = 4 ``[testenv:*]`` sections and then +listing all of them in ``envlist``. + +However, a better approach looks like this:: + + [tox] + envlist = {py26,py27}-django{15,16} + + [testenv] + basepython = + py26: python2.6 + py27: python2.7 + deps = + pytest + django15: Django>=1.5,<1.6 + django16: Django>=1.6,<1.7 + py26: unittest2 + commands = py.test + +This uses two new facilities of tox-1.8: + +- generative envlist declarations where each envname + consists of environment parts or "factors" + +- "factor" specific settings + +Let's go through this step by step. + + +Generative envlist ++++++++++++++++++++++++ + +:: + + envlist = {py26,py27}-django{15,16} + +This is bash-style syntax and will create ``2*2=4`` environment names +like this:: + + py26-django15 + py26-django16 + py27-django15 + py27-django16 + +You can still list environments explicitly along with generated ones:: + + envlist = {py26,py27}-django{15,16}, docs, flake + +.. note:: + + To help with understanding how the variants will produce section values, + you can ask tox to show their expansion with a new option:: + + $ tox -l + py26-django15 + py26-django16 + py27-django15 + py27-django16 + docs + flake + + +Factors and factor-conditional settings +++++++++++++++++++++++++++++++++++++++++ + +Parts of an environment name delimited by hyphens are called factors and can +be used to set values conditionally:: + + basepython = + py26: python2.6 + py27: python2.7 + +This conditional setting will lead to either ``python2.6`` or +``python2.7`` used as base python, e.g. ``python2.6`` is selected if current +environment contains ``py26`` factor. + +In list settings such as ``deps`` or ``commands`` you can freely intermix +optional lines with unconditional ones:: + + deps = + pytest + django15: Django>=1.5,<1.6 + django16: Django>=1.6,<1.7 + py26: unittest2 + +Reading it line by line: + +- ``pytest`` will be included unconditionally, +- ``Django>=1.5,<1.6`` will be included for environments containing ``django15`` factor, +- ``Django>=1.6,<1.7`` similarly depends on ``django16`` factor, +- ``unittest`` will be loaded for Python 2.6 environments. + +.. note:: + + Tox provides good defaults for basepython setting, so the above + ini-file can be further reduced by omitting the ``basepython`` + setting. + + +Complex factor conditions ++++++++++++++++++++++++++ + +Sometimes you need to specify same line for several factors or create a special +case for a combination of factors. Here is how you do it:: + + [tox] + envlist = py{26,27,33}-django{15,16}-{sqlite,mysql} + + [testenv] + deps = + py33-mysql: PyMySQL ; use if both py33 and mysql are in an env name + py26,py27: urllib3 ; use if any of py26 or py27 are in an env name + py{26,27}-sqlite: mock ; mocking sqlite in python 2.x + +Take a look at first ``deps`` line. It shows how you can special case something +for a combination of factors, you just join combining factors with a hyphen. +This particular line states that ``PyMySQL`` will be loaded for python 3.3, +mysql environments, e.g. ``py33-django15-mysql`` and ``py33-django16-mysql``. + +The second line shows how you use same line for several factors - by listing +them delimited by commas. It's possible to list not only simple factors, but +also their combinations like ``py26-sqlite,py27-sqlite``. + +Finally, factor expressions are expanded the same way as envlist, so last +example could be rewritten as ``py{26,27}-sqlite``. + +.. note:: + + Factors don't do substring matching against env name, instead every + hyphenated expression is split by ``-`` and if ALL the factors in an + expression are also factors of an env then that condition is considered + hold. + + For example, environment ``py26-mysql``: + + - could be matched with expressions ``py26``, ``py26-mysql``, + ``mysql-py26``, + - but not with ``py2`` or ``py26-sql``. + Other Rules and notes ===================== diff --git a/doc/example/basic.txt b/doc/example/basic.txt index 6b115c8..562e2bf 100644 --- a/doc/example/basic.txt +++ b/doc/example/basic.txt @@ -41,6 +41,7 @@ Available "default" test environments names are:: py34 jython pypy + pypy3 However, you can also create your own test environment names, see some of the examples in :doc:`examples <../examples>`. @@ -28,7 +28,7 @@ def main(): description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='1.7.2', + version='1.8.0.dev2', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff --git a/tests/test_config.py b/tests/test_config.py index bc5a683..efbbc80 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -832,6 +832,71 @@ class TestConfigTestEnv: assert conf.changedir.basename == 'testing' assert conf.changedir.dirpath().realpath() == tmpdir.realpath() + def test_factors(self, newconfig): + inisource=""" + [tox] + envlist = a-x,b + + [testenv] + deps= + dep-all + a: dep-a + b: dep-b + x: dep-x + """ + conf = newconfig([], inisource) + configs = conf.envconfigs + assert [dep.name for dep in configs['a-x'].deps] == \ + ["dep-all", "dep-a", "dep-x"] + assert [dep.name for dep in configs['b'].deps] == ["dep-all", "dep-b"] + + def test_factor_ops(self, newconfig): + inisource=""" + [tox] + envlist = {a,b}-{x,y} + + [testenv] + deps= + a,b: dep-a-or-b + a-x: dep-a-and-x + {a,b}-y: dep-ab-and-y + """ + configs = newconfig([], inisource).envconfigs + get_deps = lambda env: [dep.name for dep in configs[env].deps] + assert get_deps("a-x") == ["dep-a-or-b", "dep-a-and-x"] + assert get_deps("a-y") == ["dep-a-or-b", "dep-ab-and-y"] + assert get_deps("b-x") == ["dep-a-or-b"] + assert get_deps("b-y") == ["dep-a-or-b", "dep-ab-and-y"] + + def test_default_factors(self, newconfig): + inisource=""" + [tox] + envlist = py{26,27,33,34}-dep + + [testenv] + deps= + dep: dep + """ + conf = newconfig([], inisource) + configs = conf.envconfigs + for name, config in configs.items(): + assert config.basepython == 'python%s.%s' % (name[2], name[3]) + + @pytest.mark.issue188 + def test_factors_in_boolean(self, newconfig): + inisource=""" + [tox] + envlist = py{27,33} + + [testenv] + recreate = + py27: True + """ + configs = newconfig([], inisource).envconfigs + assert configs["py27"].recreate + assert not configs["py33"].recreate + + class TestGlobalOptions: def test_notest(self, newconfig): config = newconfig([], "") @@ -935,6 +1000,23 @@ class TestGlobalOptions: bp = "python%s.%s" %(name[2], name[3]) assert env.basepython == bp + def test_envlist_expansion(self, newconfig): + inisource = """ + [tox] + envlist = py{26,27},docs + """ + config = newconfig([], inisource) + assert config.envlist == ["py26", "py27", "docs"] + + def test_envlist_cross_product(self, newconfig): + inisource = """ + [tox] + envlist = py{26,27}-dep{1,2} + """ + config = newconfig([], inisource) + assert config.envlist == \ + ["py26-dep1", "py26-dep2", "py27-dep1", "py27-dep2"] + def test_minversion(self, tmpdir, newconfig, monkeypatch): inisource = """ [tox] diff --git a/tests/test_venv.py b/tests/test_venv.py index 6482e09..00e5f8b 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -485,9 +485,11 @@ class TestVenvTest: py.test.raises(ZeroDivisionError, "venv._pcall([1,2,3])") monkeypatch.setenv("PIP_RESPECT_VIRTUALENV", "1") monkeypatch.setenv("PIP_REQUIRE_VIRTUALENV", "1") + monkeypatch.setenv("__PYVENV_LAUNCHER__", "1") py.test.raises(ZeroDivisionError, "venv.run_install_command(['qwe'])") assert 'PIP_RESPECT_VIRTUALENV' not in os.environ assert 'PIP_REQUIRE_VIRTUALENV' not in os.environ + assert '__PYVENV_LAUNCHER__' not in os.environ def test_setenv_added_to_pcall(tmpdir, mocksession, newconfig): pkg = tmpdir.ensure("package.tar.gz") @@ -552,8 +554,6 @@ def test_run_install_command(newmocksession): assert 'install' in l[0].args env = l[0].env assert env is not None - assert 'PYTHONIOENCODING' in env - assert env['PYTHONIOENCODING'] == 'utf_8' def test_run_custom_install_command(newmocksession): mocksession = newmocksession([], """ diff --git a/tox/__init__.py b/tox/__init__.py index 530f3c4..775ed11 100644 --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '1.7.2' +__version__ = '1.8.0.dev2' class exception: class Error(Exception): diff --git a/tox/_config.py b/tox/_config.py index 1495bd8..07f2365 100644 --- a/tox/_config.py +++ b/tox/_config.py @@ -6,6 +6,7 @@ import re import shlex import string import pkg_resources +import itertools from tox.interpreters import Interpreters @@ -15,13 +16,10 @@ import tox iswin32 = sys.platform == "win32" -defaultenvs = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3'} -for _name in "py,py24,py25,py26,py27,py30,py31,py32,py33,py34".split(","): - if _name == "py": - basepython = sys.executable - else: - basepython = "python" + ".".join(_name[2:4]) - defaultenvs[_name] = basepython +default_factors = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3', + 'py': sys.executable} +for version in '24,25,26,27,30,31,32,33,34'.split(','): + default_factors['py' + version] = 'python%s.%s' % tuple(version) def parseconfig(args=None, pkg=None): if args is None: @@ -280,22 +278,19 @@ class parseini: config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None) config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") - for sectionwrapper in self._cfg: - section = sectionwrapper.name - if section.startswith(testenvprefix): - name = section[len(testenvprefix):] - envconfig = self._makeenvconfig(name, section, reader._subs, - config) - config.envconfigs[name] = envconfig - if not config.envconfigs: - config.envconfigs['python'] = \ - self._makeenvconfig("python", "_xz_9", reader._subs, config) - config.envlist = self._getenvlist(reader, toxsection) - for name in config.envlist: - if name not in config.envconfigs: - if name in defaultenvs: - config.envconfigs[name] = \ - self._makeenvconfig(name, "_xz_9", reader._subs, config) + + config.envlist, all_envs = self._getenvdata(reader, toxsection) + + # configure testenvs + known_factors = self._list_section_factors("testenv") + known_factors.update(default_factors) + known_factors.add("python") + for name in all_envs: + section = testenvprefix + name + factors = set(name.split('-')) + if section in self._cfg or factors <= known_factors: + config.envconfigs[name] = \ + self._makeenvconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs and config.envconfigs[name].develop @@ -303,10 +298,20 @@ class parseini: config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop) + def _list_section_factors(self, section): + factors = set() + if section in self._cfg: + for _, value in self._cfg[section].items(): + exprs = re.findall(r'^([\w{},-]+)\:\s+', value, re.M) + factors.update(*mapcat(_split_factor_expr, exprs)) + return factors + def _makeenvconfig(self, name, section, subs, config): vc = VenvConfig(envname=name) vc.config = config - reader = IniReader(self._cfg, fallbacksections=["testenv"]) + factors = set(name.split('-')) + reader = IniReader(self._cfg, fallbacksections=["testenv"], + factors=factors) reader.addsubstitutions(**subs) vc.develop = not config.option.installpkg and \ reader.getbool(section, "usedevelop", config.option.develop) @@ -315,10 +320,8 @@ class parseini: if reader.getdefault(section, "python", None): raise tox.exception.ConfigError( "'python=' key was renamed to 'basepython='") - if name in defaultenvs: - bp = defaultenvs[name] - else: - bp = sys.executable + bp = next((default_factors[f] for f in factors if f in default_factors), + sys.executable) vc.basepython = reader.getdefault(section, "basepython", bp) vc._basepython_info = config.interpreters.get_info(vc.basepython) reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname, @@ -386,20 +389,25 @@ class parseini: "'install_command' must contain '{packages}' substitution") return vc - def _getenvlist(self, reader, toxsection): - env = self.config.option.env - if not env: - env = os.environ.get("TOXENV", None) - if not env: - envlist = reader.getlist(toxsection, "envlist", sep=",") - if not envlist: - envlist = self.config.envconfigs.keys() - return envlist - envlist = _split_env(env) - if "ALL" in envlist: - envlist = list(self.config.envconfigs) - envlist.sort() - return envlist + def _getenvdata(self, reader, toxsection): + envstr = self.config.option.env \ + or os.environ.get("TOXENV") \ + or reader.getdefault(toxsection, "envlist", replace=False) \ + or [] + envlist = _split_env(envstr) + + # collect section envs + all_envs = set(envlist) - set(["ALL"]) + for section in self._cfg: + if section.name.startswith(testenvprefix): + all_envs.add(section.name[len(testenvprefix):]) + if not all_envs: + all_envs.add("python") + + if not envlist or "ALL" in envlist: + envlist = sorted(all_envs) + + return envlist, all_envs def _replace_forced_dep(self, name, config): """ @@ -427,17 +435,32 @@ class parseini: dep2_name = pkg_resources.Requirement.parse(dep2).project_name return dep1_name == dep2_name + def _split_env(env): """if handed a list, action="append" was used for -e """ - envlist = [] if not isinstance(env, list): env = [env] - for to_split in env: - for single_env in to_split.split(","): - # "remove True or", if not allowing multiple same runs, update tests - if True or single_env not in envlist: - envlist.append(single_env) - return envlist + return mapcat(_expand_envstr, env) + +def _split_factor_expr(expr): + partial_envs = _expand_envstr(expr) + return [set(e.split('-')) for e in partial_envs] + +def _expand_envstr(envstr): + # split by commas not in groups + tokens = re.split(r'(\{[^}]+\})|,', envstr) + envlist = [''.join(g).strip() + for k, g in itertools.groupby(tokens, key=bool) if k] + + def expand(env): + tokens = re.split(r'\{([^}]+)\}', env) + parts = [token.split(',') for token in tokens] + return [''.join(variant) for variant in itertools.product(*parts)] + + return mapcat(expand, envlist) + +def mapcat(f, seq): + return list(itertools.chain.from_iterable(map(f, seq))) class DepConfig: def __init__(self, name, indexserver=None): @@ -468,9 +491,10 @@ RE_ITEM_REF = re.compile( class IniReader: - def __init__(self, cfgparser, fallbacksections=None): + def __init__(self, cfgparser, fallbacksections=None, factors=()): self._cfg = cfgparser self.fallbacksections = fallbacksections or [] + self.factors = factors self._subs = {} self._subststack = [] @@ -572,9 +596,12 @@ class IniReader: def getbool(self, section, name, default=None): s = self.getdefault(section, name, default) + if not s: + s = default if s is None: raise KeyError("no config value [%s] %s found" % ( section, name)) + if not isinstance(s, bool): if s.lower() == "true": s = True @@ -586,18 +613,19 @@ class IniReader: return s def getdefault(self, section, name, default=None, replace=True): - try: - x = self._cfg[section][name] - except KeyError: - for fallbacksection in self.fallbacksections: - try: - x = self._cfg[fallbacksection][name] - except KeyError: - pass - else: - break - else: - x = default + x = None + for s in [section] + self.fallbacksections: + try: + x = self._cfg[s][name] + break + except KeyError: + continue + + if x is None: + x = default + else: + x = self._apply_factors(x) + if replace and x and hasattr(x, 'replace'): self._subststack.append((section, name)) try: @@ -607,6 +635,19 @@ class IniReader: #print "getdefault", section, name, "returned", repr(x) return x + def _apply_factors(self, s): + def factor_line(line): + m = re.search(r'^([\w{},-]+)\:\s+(.+)', line) + if not m: + return line + + expr, line = m.groups() + if any(fs <= self.factors for fs in _split_factor_expr(expr)): + return line + + lines = s.strip().splitlines() + return '\n'.join(filter(None, map(factor_line, lines))) + def _replace_env(self, match): match_value = match.group('substitution_value') if not match_value: diff --git a/tox/_venv.py b/tox/_venv.py index df37eac..0c56cc0 100644 --- a/tox/_venv.py +++ b/tox/_venv.py @@ -1,5 +1,6 @@ from __future__ import with_statement import sys, os +import codecs import py import tox from tox._config import DepConfig @@ -271,16 +272,19 @@ class VirtualEnv(object): if '{opts}' in argv: i = argv.index('{opts}') argv[i:i+1] = list(options) - for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV'): + for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV', + '__PYVENV_LAUNCHER__'): try: del os.environ[x] except KeyError: pass - env = dict(PYTHONIOENCODING='utf_8') - if extraenv is not None: - env.update(extraenv) + old_stdout = sys.stdout + sys.stdout = codecs.getwriter('utf8')(sys.stdout) + if extraenv is None: + extraenv = {} self._pcall(argv, cwd=self.envconfig.config.toxinidir, - extraenv=env, action=action) + extraenv=extraenv, action=action) + sys.stdout = old_stdout def _install(self, deps, extraopts=None, action=None): if not deps: |