diff options
| -rw-r--r-- | .hgignore | 1 | ||||
| -rw-r--r--[-rwxr-xr-x] | CHANGELOG | 46 | ||||
| -rw-r--r-- | CONTRIBUTORS | 6 | ||||
| -rw-r--r-- | doc/announce/release-1.8.txt | 54 | ||||
| -rw-r--r-- | doc/conf.py | 3 | ||||
| -rw-r--r-- | doc/config-v2.txt | 2 | ||||
| -rw-r--r-- | doc/config.txt | 191 | ||||
| -rw-r--r-- | doc/example/basic.txt | 17 | ||||
| -rw-r--r-- | doc/example/jenkins.txt | 7 | ||||
| -rwxr-xr-x | doc/example/pytest.txt | 2 | ||||
| -rw-r--r-- | doc/index.txt | 4 | ||||
| -rw-r--r-- | setup.py | 6 | ||||
| -rw-r--r-- | tests/conftest.py | 2 | ||||
| -rw-r--r-- | tests/test_config.py | 194 | ||||
| -rw-r--r-- | tests/test_interpreters.py | 2 | ||||
| -rw-r--r-- | tests/test_quickstart.py | 180 | ||||
| -rw-r--r-- | tests/test_venv.py | 22 | ||||
| -rw-r--r-- | tests/test_z_cmdline.py | 35 | ||||
| -rw-r--r-- | tox.ini | 6 | ||||
| -rw-r--r-- | tox/__init__.py | 4 | ||||
| -rw-r--r-- | tox/_cmdline.py | 29 | ||||
| -rw-r--r-- | tox/_config.py | 248 | ||||
| -rw-r--r-- | tox/_quickstart.py | 2 | ||||
| -rw-r--r-- | tox/_venv.py | 23 | ||||
| -rw-r--r-- | tox/_verlib.py | 1 | ||||
| -rw-r--r-- | tox/interpreters.py | 4 |
26 files changed, 898 insertions, 193 deletions
@@ -14,3 +14,4 @@ dist doc/_build/ tox.egg-info .tox +.cache diff --git a/CHANGELOG b/CHANGELOG index e6168cb..3e73166 100755..100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,49 @@ +1.8.1.dev +----------- + +- fix issue190: allow setenv to be empty. + + +1.8.0 +----------- + +- new multi-dimensional configuration support. Many thanks to + Alexander Schepanovski for the complete PR with docs. + And to Mike Bayer and others for testing and feedback. + +- fix issue148: remove "__PYVENV_LAUNCHER__" from os.environ when starting + subprocesses. Thanks Steven Myint. + +- fix issue152: set VIRTUAL_ENV when running test commands, + thanks Florian Ludwig. + +- better report if we can't get version_info from an interpreter + executable. Thanks Floris Bruynooghe. + + +1.7.2 +----------- + +- fix issue150: parse {posargs} more like we used to do it pre 1.7.0. + The 1.7.0 behaviour broke a lot of OpenStack projects. + See PR85 and the issue discussions for (far) more details, hopefully + resulting in a more refined behaviour in the 1.8 series. + And thanks to Clark Boylan for the PR. + +- fix issue59: add a config variable ``skip-missing-interpreters`` as well as + command line option ``--skip-missing-interpreters`` which won't fail the + build if Python interpreters listed in tox.ini are missing. Thanks + Alexandre Conrad for PR104. + +- fix issue164: better traceback info in case of failing test commands. + Thanks Marc Abramowitz for PR92. + +- support optional env variable substitution, thanks Morgan Fainberg + for PR86. + +- limit python hashseed to 1024 on Windows to prevent possible + memory errors. Thanks March Schlaich for the PR90. + 1.7.1 --------- diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 0244c2b..ef8576d 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -3,6 +3,7 @@ contributions: Krisztian Fekete Marc Abramowitz +Aleaxner Schepanovski Sridhar Ratnakumar Barry Warsaw Chris Rose @@ -23,3 +24,8 @@ Matt Good Mattieu Agopian Asmund Grammeltwedt Ionel Maries Cristian +Alexandre Conrad +Morgan Fainberg +Marc Schlaich +Clark Boylan +Eugene Yunak diff --git a/doc/announce/release-1.8.txt b/doc/announce/release-1.8.txt new file mode 100644 index 0000000..b8a2218 --- /dev/null +++ b/doc/announce/release-1.8.txt @@ -0,0 +1,54 @@ +tox 1.8: Generative/combinatorial environments specs +============================================================================= + +I am happy to announce tox 1.8 which implements parametrized environments. + +See https://tox.testrun.org/latest/config.html#generating-environments-conditional-settings +for examples and the new backward compatible syntax in your tox.ini file. + +Many thanks to Alexander Schepanovski for implementing and refining +it based on the specifcation draft. + +More documentation about tox in general: + + http://tox.testrun.org/ + +Installation: + + pip install -U tox + +code hosting and issue tracking on bitbucket: + + https://bitbucket.org/hpk42/tox + +What is tox? +---------------- + +tox standardizes and automates tedious test activities driven from a +simple ``tox.ini`` file, including: + +* creation and management of different virtualenv environments + with different Python interpreters +* packaging and installing your package into each of them +* running your test tool of choice, be it nose, py.test or unittest2 or other tools such as "sphinx" doc checks +* testing dev packages against each other without needing to upload to PyPI + +best, +Holger Krekel, merlinux GmbH + + +Changes 1.8 (compared to 1.7.2) +--------------------------------------- + +- new multi-dimensional configuration support. Many thanks to + Alexander Schepanovski for the complete PR with docs. + And to Mike Bayer and others for testing and feedback. + +- fix issue148: remove "__PYVENV_LAUNCHER__" from os.environ when starting + subprocesses. Thanks Steven Myint. + +- fix issue152: set VIRTUAL_ENV when running test commands, + thanks Florian Ludwig. + +- better report if we can't get version_info from an interpreter + executable. Thanks Floris Bruynooghe. diff --git a/doc/conf.py b/doc/conf.py index 76dc4c3..92ed895 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -48,7 +48,8 @@ copyright = u'2013, holger krekel and others' # built documents. # # The short X.Y version. -release = version = "1.7.1" +release = "1.8" +version = "1.8.0" # The full version, including alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation diff --git a/doc/config-v2.txt b/doc/config-v2.txt index cf4cf3b..42973c4 100644 --- a/doc/config-v2.txt +++ b/doc/config-v2.txt @@ -195,7 +195,7 @@ Default settings related to environments names/variants tox comes with predefined settings for certain variants, namely: * ``{easy,pip}`` use easy_install or pip respectively -* ``{py24,py25,py26,py27,py31,py32,py33,pypy19]`` use the respective +* ``{py24,py25,py26,py27,py31,py32,py33,py34,pypy19]`` use the respective pythonNN or PyPy interpreter * ``{win32,linux,darwin}`` defines the according ``platform``. diff --git a/doc/config.txt b/doc/config.txt index 054400d..8ce64db 100644 --- a/doc/config.txt +++ b/doc/config.txt @@ -32,16 +32,27 @@ and will first lookup global tox settings in this section:: ... # override [tox] settings for the jenkins context # note: for jenkins distshare defaults to ``{toxworkdir}/distshare``. +.. confval:: skip_missing_interpreters=BOOL -envlist setting -+++++++++++++++ + .. versionadded:: 1.7.2 -Determining the environment list that ``tox`` is to operate on -happens in this order: + Setting this to ``True`` is equivalent of passing the + ``--skip-missing-interpreters`` command line option, and will force ``tox`` to + return success even if some of the specified environments were missing. This is + useful for some CI systems or running on a developer box, where you might only + have a subset of all your supported interpreters installed but don't want to + mark the build as failed because of it. As expected, the command line switch + always overrides this setting if passed on the invokation. + **Default:** ``False`` -* command line option ``-eENVLIST`` -* environment variable ``TOXENV`` -* ``tox.ini`` file's ``envlist`` +.. confval:: envlist=CSV + + Determining the environment list that ``tox`` is to operate on + happens in this order (if any is found, no further lookups are made): + + * command line option ``-eENVLIST`` + * environment variable ``TOXENV`` + * ``tox.ini`` file's ``envlist`` Virtualenv test environment settings @@ -284,6 +295,26 @@ then the value will be retrieved as ``os.environ['KEY']`` and raise an Error if the environment variable does not exist. + +environment variable substitutions with default values +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +If you specify a substitution string like this:: + + {env:KEY:DEFAULTVALUE} + +then the value will be retrieved as ``os.environ['KEY']`` +and replace with DEFAULTVALUE if the environment variable does not +exist. + +If you specify a substitution string like this:: + + {env:KEY:} + +then the value will be retrieved as ``os.environ['KEY']`` +and replace with and empty string if the environment variable does not +exist. + .. _`command positional substitution`: .. _`positional substitution`: @@ -351,6 +382,152 @@ You can put default values in one section and reference them in others to avoid {[base]deps} +Generating environments, conditional settings +--------------------------------------------- + +.. versionadded:: 1.8 + +Suppose you want to test your package against python2.6, python2.7 and against +several versions of a dependency, say Django 1.5 and Django 1.6. You can +accomplish that by writing down 2*2 = 4 ``[testenv:*]`` sections and then +listing all of them in ``envlist``. + +However, a better approach looks like this:: + + [tox] + envlist = {py26,py27}-django{15,16} + + [testenv] + basepython = + py26: python2.6 + py27: python2.7 + deps = + pytest + django15: Django>=1.5,<1.6 + django16: Django>=1.6,<1.7 + py26: unittest2 + commands = py.test + +This uses two new facilities of tox-1.8: + +- generative envlist declarations where each envname + consists of environment parts or "factors" + +- "factor" specific settings + +Let's go through this step by step. + + +Generative envlist ++++++++++++++++++++++++ + +:: + + envlist = {py26,py27}-django{15,16} + +This is bash-style syntax and will create ``2*2=4`` environment names +like this:: + + py26-django15 + py26-django16 + py27-django15 + py27-django16 + +You can still list environments explicitly along with generated ones:: + + envlist = {py26,py27}-django{15,16}, docs, flake + +.. note:: + + To help with understanding how the variants will produce section values, + you can ask tox to show their expansion with a new option:: + + $ tox -l + py26-django15 + py26-django16 + py27-django15 + py27-django16 + docs + flake + + +Factors and factor-conditional settings +++++++++++++++++++++++++++++++++++++++++ + +Parts of an environment name delimited by hyphens are called factors and can +be used to set values conditionally:: + + basepython = + py26: python2.6 + py27: python2.7 + +This conditional setting will lead to either ``python2.6`` or +``python2.7`` used as base python, e.g. ``python2.6`` is selected if current +environment contains ``py26`` factor. + +In list settings such as ``deps`` or ``commands`` you can freely intermix +optional lines with unconditional ones:: + + deps = + pytest + django15: Django>=1.5,<1.6 + django16: Django>=1.6,<1.7 + py26: unittest2 + +Reading it line by line: + +- ``pytest`` will be included unconditionally, +- ``Django>=1.5,<1.6`` will be included for environments containing ``django15`` factor, +- ``Django>=1.6,<1.7`` similarly depends on ``django16`` factor, +- ``unittest`` will be loaded for Python 2.6 environments. + +.. note:: + + Tox provides good defaults for basepython setting, so the above + ini-file can be further reduced by omitting the ``basepython`` + setting. + + +Complex factor conditions ++++++++++++++++++++++++++ + +Sometimes you need to specify same line for several factors or create a special +case for a combination of factors. Here is how you do it:: + + [tox] + envlist = py{26,27,33}-django{15,16}-{sqlite,mysql} + + [testenv] + deps = + py33-mysql: PyMySQL ; use if both py33 and mysql are in an env name + py26,py27: urllib3 ; use if any of py26 or py27 are in an env name + py{26,27}-sqlite: mock ; mocking sqlite in python 2.x + +Take a look at first ``deps`` line. It shows how you can special case something +for a combination of factors, you just join combining factors with a hyphen. +This particular line states that ``PyMySQL`` will be loaded for python 3.3, +mysql environments, e.g. ``py33-django15-mysql`` and ``py33-django16-mysql``. + +The second line shows how you use same line for several factors - by listing +them delimited by commas. It's possible to list not only simple factors, but +also their combinations like ``py26-sqlite,py27-sqlite``. + +Finally, factor expressions are expanded the same way as envlist, so last +example could be rewritten as ``py{26,27}-sqlite``. + +.. note:: + + Factors don't do substring matching against env name, instead every + hyphenated expression is split by ``-`` and if ALL the factors in an + expression are also factors of an env then that condition is considered + hold. + + For example, environment ``py26-mysql``: + + - could be matched with expressions ``py26``, ``py26-mysql``, + ``mysql-py26``, + - but not with ``py2`` or ``py26-sql``. + Other Rules and notes ===================== diff --git a/doc/example/basic.txt b/doc/example/basic.txt index 19a15d4..562e2bf 100644 --- a/doc/example/basic.txt +++ b/doc/example/basic.txt @@ -1,4 +1,3 @@ - Basic usage ============================================= @@ -13,6 +12,7 @@ reside next to your ``setup.py`` file:: [tox] envlist = py26,py27 [testenv] + deps=pytest # or 'nose' or ... commands=py.test # or 'nosetests' or ... To sdist-package, install and test your project, you can @@ -38,8 +38,10 @@ Available "default" test environments names are:: py31 py32 py33 + py34 jython pypy + pypy3 However, you can also create your own test environment names, see some of the examples in :doc:`examples <../examples>`. @@ -207,6 +209,10 @@ a test run when ``python setup.py test`` is issued:: import sys class Tox(TestCommand): + user_options = [('tox-args=', 'a', "Arguments to pass to tox")] + def initialize_options(self): + TestCommand.initialize_options(self) + self.tox_args = None def finalize_options(self): TestCommand.finalize_options(self) self.test_args = [] @@ -214,7 +220,8 @@ a test run when ``python setup.py test`` is issued:: def run_tests(self): #import here, cause outside the eggs aren't loaded import tox - errno = tox.cmdline(self.test_args) + import shlex + errno = tox.cmdline(args=shlex.split(self.tox_args)) sys.exit(errno) setup( @@ -227,5 +234,9 @@ Now if you run:: python setup.py test -this will install tox and then run tox. +this will install tox and then run tox. You can pass arguments to ``tox`` +using the ``--tox-args`` or ``-a`` command-line options. For example:: + + python setup.py test -a "-epy27" +is equivalent to running ``tox -epy27``. diff --git a/doc/example/jenkins.txt b/doc/example/jenkins.txt index bfdceee..2d8dc25 100644 --- a/doc/example/jenkins.txt +++ b/doc/example/jenkins.txt @@ -32,7 +32,9 @@ using these steps: The last point requires that your test command creates JunitXML files, for example with ``py.test`` it is done like this: - commands=py.test --junitxml=junit-{envname}.xml +.. code-block:: ini + + commands = py.test --junitxml=junit-{envname}.xml @@ -57,7 +59,7 @@ with this:: exec urllib.urlopen(url).read() in d d['cmdline'](['--recreate']) -The downloaded `toxbootstrap.py`_ file downloads all neccessary files to +The downloaded `toxbootstrap.py` file downloads all neccessary files to install ``tox`` in a virtual sub environment. Notes: * uncomment the line containing ``USETOXDEV`` to use the latest @@ -68,7 +70,6 @@ install ``tox`` in a virtual sub environment. Notes: will cause tox to reinstall all virtual environments all the time which is often what one wants in CI server contexts) -.. _`toxbootstrap.py`: https://bitbucket.org/hpk42/tox/raw/default/toxbootstrap.py Integrating "sphinx" documentation checks in a Jenkins job ---------------------------------------------------------------- diff --git a/doc/example/pytest.txt b/doc/example/pytest.txt index cba8741..6f98cf3 100755 --- a/doc/example/pytest.txt +++ b/doc/example/pytest.txt @@ -105,7 +105,7 @@ directories; pytest will still find and import them by adding their parent directory to ``sys.path`` but they won't be copied to other places or be found by Python's import system outside of pytest. -.. _`fully qualified name`: http://pytest.org/latest/goodpractises.html#package-name +.. _`fully qualified name`: http://pytest.org/latest/goodpractises.html#test-package-name .. include:: ../links.txt diff --git a/doc/index.txt b/doc/index.txt index c1baae2..17d063d 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -68,8 +68,8 @@ Current features support for configuring the installer command through :confval:`install_command=ARGV`. -* **cross-Python compatible**: Python-2.6 up to Python-3.3, - Jython and pypy_ support. +* **cross-Python compatible**: CPython-2.6, 2.7, 3.2 and higher, + Jython and pypy_. * **cross-platform**: Windows and Unix style environments @@ -19,16 +19,14 @@ class Tox(TestCommand): def main(): version = sys.version_info[:2] install_requires = ['virtualenv>=1.11.2', 'py>=1.4.17', ] - if version < (2, 7) or (3, 0) <= version <= (3, 1): + if version < (2, 7): install_requires += ['argparse'] - if version < (2,6): - install_requires += ["simplejson"] setup( name='tox', description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='1.7.1', + version='1.8.1.dev1', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff --git a/tests/conftest.py b/tests/conftest.py index 1d4bdde..3e2493b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,2 +1,2 @@ -from tox._pytestplugin import * +from tox._pytestplugin import * # noqa diff --git a/tests/test_config.py b/tests/test_config.py index c4ec9cc..c823cb3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,12 +1,11 @@ -import tox -import pytest -import os, sys -import subprocess +import sys from textwrap import dedent import py +import pytest +import tox import tox._config -from tox._config import * +from tox._config import * # noqa from tox._config import _split_env @@ -110,7 +109,6 @@ class TestConfigPackage: def test_defaults_distshare(self, tmpdir, newconfig): config = newconfig([], "") - envconfig = config.envconfigs['python'] assert config.distshare == config.homedir.join(".tox", "distshare") def test_defaults_changed_dir(self, tmpdir, newconfig): @@ -168,6 +166,7 @@ class TestIniParser: key2={xyz} """) reader = IniReader(config._cfg, fallbacksections=['mydefault']) + assert reader is not None py.test.raises(tox.exception.ConfigError, 'reader.getdefault("mydefault", "key2")') @@ -242,6 +241,22 @@ class TestIniParser: py.test.raises(tox.exception.ConfigError, 'reader.getdefault("section", "key2")') + def test_getdefault_environment_substitution_with_default(self, monkeypatch, newconfig): + monkeypatch.setenv("KEY1", "hello") + config = newconfig(""" + [section] + key1={env:KEY1:DEFAULT_VALUE} + key2={env:KEY2:DEFAULT_VALUE} + key3={env:KEY3:} + """) + reader = IniReader(config._cfg) + x = reader.getdefault("section", "key1") + assert x == "hello" + x = reader.getdefault("section", "key2") + assert x == "DEFAULT_VALUE" + x = reader.getdefault("section", "key3") + assert x == "" + def test_getdefault_other_section_substitution(self, newconfig): config = newconfig(""" [section] @@ -278,7 +293,7 @@ class TestIniParser: # "reader.getargvlist('section', 'key1')") assert reader.getargvlist('section', 'key1') == [] x = reader.getargvlist("section", "key2") - assert x == [["cmd1", "with space", "grr"], + assert x == [["cmd1", "with", "space", "grr"], ["cmd2", "grr"]] def test_argvlist_windows_escaping(self, tmpdir, newconfig): @@ -304,7 +319,7 @@ class TestIniParser: # "reader.getargvlist('section', 'key1')") assert reader.getargvlist('section', 'key1') == [] x = reader.getargvlist("section", "key2") - assert x == [["cmd1", "with space", "grr"]] + assert x == [["cmd1", "with", "space", "grr"]] def test_argvlist_quoting_in_command(self, tmpdir, newconfig): @@ -346,6 +361,34 @@ class TestIniParser: assert argvlist[0] == ["cmd1"] assert argvlist[1] == ["cmd2", "value2", "other"] + def test_argvlist_quoted_posargs(self, tmpdir, newconfig): + config = newconfig(""" + [section] + key2= + cmd1 --foo-args='{posargs}' + cmd2 -f '{posargs}' + cmd3 -f {posargs} + """) + reader = IniReader(config._cfg) + reader.addsubstitutions(["foo", "bar"]) + assert reader.getargvlist('section', 'key1') == [] + x = reader.getargvlist("section", "key2") + assert x == [["cmd1", "--foo-args=foo bar"], + ["cmd2", "-f", "foo bar"], + ["cmd3", "-f", "foo", "bar"]] + + def test_argvlist_posargs_with_quotes(self, tmpdir, newconfig): + config = newconfig(""" + [section] + key2= + cmd1 -f {posargs} + """) + reader = IniReader(config._cfg) + reader.addsubstitutions(["foo", "'bar", "baz'"]) + assert reader.getargvlist('section', 'key1') == [] + x = reader.getargvlist("section", "key2") + assert x == [["cmd1", "-f", "foo", "bar baz"]] + def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig): config = newconfig(""" @@ -511,7 +554,7 @@ class TestConfigTestEnv: envconfig = config.envconfigs['python'] assert envconfig.envpython == envconfig.envbindir.join("python") - @pytest.mark.parametrize("bp", ["jython", "pypy"]) + @pytest.mark.parametrize("bp", ["jython", "pypy", "pypy3"]) def test_envbindir_jython(self, tmpdir, newconfig, bp): config = newconfig(""" [testenv] @@ -806,6 +849,100 @@ class TestConfigTestEnv: assert conf.changedir.basename == 'testing' assert conf.changedir.dirpath().realpath() == tmpdir.realpath() + def test_factors(self, newconfig): + inisource=""" + [tox] + envlist = a-x,b + + [testenv] + deps= + dep-all + a: dep-a + b: dep-b + x: dep-x + """ + conf = newconfig([], inisource) + configs = conf.envconfigs + assert [dep.name for dep in configs['a-x'].deps] == \ + ["dep-all", "dep-a", "dep-x"] + assert [dep.name for dep in configs['b'].deps] == ["dep-all", "dep-b"] + + def test_factor_ops(self, newconfig): + inisource=""" + [tox] + envlist = {a,b}-{x,y} + + [testenv] + deps= + a,b: dep-a-or-b + a-x: dep-a-and-x + {a,b}-y: dep-ab-and-y + """ + configs = newconfig([], inisource).envconfigs + get_deps = lambda env: [dep.name for dep in configs[env].deps] + assert get_deps("a-x") == ["dep-a-or-b", "dep-a-and-x"] + assert get_deps("a-y") == ["dep-a-or-b", "dep-ab-and-y"] + assert get_deps("b-x") == ["dep-a-or-b"] + assert get_deps("b-y") == ["dep-a-or-b", "dep-ab-and-y"] + + def test_default_factors(self, newconfig): + inisource=""" + [tox] + envlist = py{26,27,33,34}-dep + + [testenv] + deps= + dep: dep + """ + conf = newconfig([], inisource) + configs = conf.envconfigs + for name, config in configs.items(): + assert config.basepython == 'python%s.%s' % (name[2], name[3]) + + @pytest.mark.issue188 + def test_factors_in_boolean(self, newconfig): + inisource=""" + [tox] + envlist = py{27,33} + + [testenv] + recreate = + py27: True + """ + configs = newconfig([], inisource).envconfigs + assert configs["py27"].recreate + assert not configs["py33"].recreate + + @pytest.mark.issue190 + def test_factors_in_setenv(self, newconfig): + inisource=""" + [tox] + envlist = py27,py26 + + [testenv] + setenv = + py27: X = 1 + """ + configs = newconfig([], inisource).envconfigs + assert configs["py27"].setenv["X"] == "1" + assert "X" not in configs["py26"].setenv + + def test_period_in_factor(self, newconfig): + inisource=""" + [tox] + envlist = py27-{django1.6,django1.7} + + [testenv] + deps = + django1.6: Django==1.6 + django1.7: Django==1.7 + """ + configs = newconfig([], inisource).envconfigs + assert sorted(configs) == ["py27-django1.6", "py27-django1.7"] + assert [d.name for d in configs["py27-django1.6"].deps] \ + == ["Django==1.6"] + + class TestGlobalOptions: def test_notest(self, newconfig): config = newconfig([], "") @@ -890,7 +1027,7 @@ class TestGlobalOptions: assert str(env.basepython) == sys.executable def test_default_environments(self, tmpdir, newconfig, monkeypatch): - envs = "py26,py27,py31,py32,py33,jython,pypy" + envs = "py26,py27,py31,py32,py33,py34,jython,pypy,pypy3" inisource = """ [tox] envlist = %s @@ -902,13 +1039,30 @@ class TestGlobalOptions: env = config.envconfigs[name] if name == "jython": assert env.basepython == "jython" - elif name == "pypy": - assert env.basepython == "pypy" + elif name.startswith("pypy"): + assert env.basepython == name else: assert name.startswith("py") bp = "python%s.%s" %(name[2], name[3]) assert env.basepython == bp + def test_envlist_expansion(self, newconfig): + inisource = """ + [tox] + envlist = py{26,27},docs + """ + config = newconfig([], inisource) + assert config.envlist == ["py26", "py27", "docs"] + + def test_envlist_cross_product(self, newconfig): + inisource = """ + [tox] + envlist = py{26,27}-dep{1,2} + """ + config = newconfig([], inisource) + assert config.envlist == \ + ["py26-dep1", "py26-dep2", "py27-dep1", "py27-dep2"] + def test_minversion(self, tmpdir, newconfig, monkeypatch): inisource = """ [tox] @@ -917,6 +1071,22 @@ class TestGlobalOptions: config = newconfig([], inisource) assert config.minversion == "3.0" + def test_skip_missing_interpreters_true(self, tmpdir, newconfig, monkeypatch): + inisource = """ + [tox] + skip_missing_interpreters = True + """ + config = newconfig([], inisource) + assert config.option.skip_missing_interpreters + + def test_skip_missing_interpreters_false(self, tmpdir, newconfig, monkeypatch): + inisource = """ + [tox] + skip_missing_interpreters = False + """ + config = newconfig([], inisource) + assert not config.option.skip_missing_interpreters + def test_defaultenv_commandline(self, tmpdir, newconfig, monkeypatch): config = newconfig(["-epy24"], "") env = config.envconfigs['py24'] diff --git a/tests/test_interpreters.py b/tests/test_interpreters.py index 39eabc6..f06a69a 100644 --- a/tests/test_interpreters.py +++ b/tests/test_interpreters.py @@ -2,7 +2,7 @@ import sys import os import pytest -from tox.interpreters import * +from tox.interpreters import * # noqa @pytest.fixture def interpreters(): diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index c376d4b..df8a98f 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -6,6 +6,7 @@ import tox._quickstart def cleandir(tmpdir): tmpdir.chdir() + class TestToxQuickstartMain(object): def mock_term_input_return_values(self, return_values): @@ -23,12 +24,26 @@ class TestToxQuickstartMain(object): return mock_term_input - def test_quickstart_main_choose_individual_pythons_and_pytest(self, - monkeypatch): + def test_quickstart_main_choose_individual_pythons_and_pytest( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', self.get_mock_term_input( - ['4', 'Y', 'Y', 'Y', 'Y', 'Y', 'N', 'py.test', 'pytest'])) + [ + '4', # Python versions: choose one by one + 'Y', # py26 + 'Y', # py27 + 'Y', # py32 + 'Y', # py33 + 'Y', # py34 + 'Y', # pypy + 'N', # jython + 'py.test', # command to run tests + 'pytest' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -39,7 +54,7 @@ class TestToxQuickstartMain(object): # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy +envlist = py26, py27, py32, py33, py34, pypy [testenv] commands = py.test @@ -49,11 +64,26 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_individual_pythons_and_nose_adds_deps(self, monkeypatch): + def test_quickstart_main_choose_individual_pythons_and_nose_adds_deps( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['4', 'Y', 'Y', 'Y', 'Y', 'Y', 'N', - 'nosetests', ''])) + self.get_mock_term_input( + [ + '4', # Python versions: choose one by one + 'Y', # py26 + 'Y', # py27 + 'Y', # py32 + 'Y', # py33 + 'Y', # py34 + 'Y', # pypy + 'N', # jython + 'nosetests', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -64,7 +94,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy +envlist = py26, py27, py32, py33, py34, pypy [testenv] commands = nosetests @@ -74,11 +104,26 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_individual_pythons_and_trial_adds_deps(self, monkeypatch): + def test_quickstart_main_choose_individual_pythons_and_trial_adds_deps( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['4', 'Y', 'Y', 'Y', 'Y', 'Y', 'N', - 'trial', ''])) + self.get_mock_term_input( + [ + '4', # Python versions: choose one by one + 'Y', # py26 + 'Y', # py27 + 'Y', # py32 + 'Y', # py33 + 'Y', # py34 + 'Y', # pypy + 'N', # jython + 'trial', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -89,7 +134,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy +envlist = py26, py27, py32, py33, py34, pypy [testenv] commands = trial @@ -99,11 +144,26 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_individual_pythons_and_pytest_adds_deps(self, monkeypatch): + def test_quickstart_main_choose_individual_pythons_and_pytest_adds_deps( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['4', 'Y', 'Y', 'Y', 'Y', 'Y', 'N', - 'py.test', ''])) + self.get_mock_term_input( + [ + '4', # Python versions: choose one by one + 'Y', # py26 + 'Y', # py27 + 'Y', # py32 + 'Y', # py33 + 'Y', # py34 + 'Y', # pypy + 'N', # jython + 'py.test', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) expected_tox_ini = """ @@ -113,7 +173,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy +envlist = py26, py27, py32, py33, py34, pypy [testenv] commands = py.test @@ -123,10 +183,19 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_py27_and_pytest_adds_deps(self, monkeypatch): + def test_quickstart_main_choose_py27_and_pytest_adds_deps( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['1', 'py.test', ''])) + self.get_mock_term_input( + [ + '1', # py27 + 'py.test', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -147,10 +216,19 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_py27_and_py33_and_pytest_adds_deps(self, monkeypatch): + def test_quickstart_main_choose_py27_and_py33_and_pytest_adds_deps( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['2', 'py.test', ''])) + self.get_mock_term_input( + [ + '2', # py27 and py33 + 'py.test', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -171,10 +249,19 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_all_pythons_and_pytest_adds_deps(self, monkeypatch): + def test_quickstart_main_choose_all_pythons_and_pytest_adds_deps( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['3', 'py.test', ''])) + self.get_mock_term_input( + [ + '3', # all Python versions + 'py.test', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -185,7 +272,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy, jython +envlist = py26, py27, py32, py33, py34, pypy, jython [testenv] commands = py.test @@ -195,10 +282,26 @@ deps = result = open('tox.ini').read() assert(result == expected_tox_ini) - def test_quickstart_main_choose_individual_pythons_and_defaults(self, monkeypatch): + def test_quickstart_main_choose_individual_pythons_and_defaults( + self, + monkeypatch): monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['4', '', '', '', '', '', '', '', '', '', ''])) + self.get_mock_term_input( + [ + '4', # Python versions: choose one by one + '', # py26 + '', # py27 + '', # py32 + '', # py33 + '', # py34 + '', # pypy + '', # jython + '', # command to run tests + '' # test dependencies + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -209,7 +312,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy, jython +envlist = py26, py27, py32, py33, py34, pypy, jython [testenv] commands = {envpython} setup.py test @@ -228,7 +331,22 @@ deps = monkeypatch.setattr( tox._quickstart, 'term_input', - self.get_mock_term_input(['4', '', '', '', '', '', '', '', '', '', '', ''])) + self.get_mock_term_input( + [ + '4', # Python versions: choose one by one + '', # py26 + '', # py27 + '', # py32 + '', # py33 + '', # py34 + '', # pypy + '', # jython + '', # command to run tests + '', # test dependencies + '', # tox.ini already exists; overwrite? + ] + ) + ) tox._quickstart.main(argv=['tox-quickstart']) @@ -239,7 +357,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy, jython +envlist = py26, py27, py32, py33, py34, pypy, jython [testenv] commands = {envpython} setup.py test @@ -257,6 +375,7 @@ class TestToxQuickstart(object): 'py27': True, 'py32': True, 'py33': True, + 'py34': True, 'pypy': True, 'commands': 'py.test', 'deps': 'pytest', @@ -268,7 +387,7 @@ class TestToxQuickstart(object): # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32, py33, pypy +envlist = py26, py27, py32, py33, py34, pypy [testenv] commands = py.test @@ -339,6 +458,7 @@ deps = 'py27': True, 'py32': True, 'py33': True, + 'py34': True, 'pypy': True, 'commands': 'nosetests -v', 'deps': 'nose', @@ -350,7 +470,7 @@ deps = # and then run "tox" from this directory. [tox] -envlist = py27, py32, py33, pypy +envlist = py27, py32, py33, py34, pypy [testenv] commands = nosetests -v diff --git a/tests/test_venv.py b/tests/test_venv.py index 07b89f4..b1d1f2e 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -3,7 +3,8 @@ import tox import pytest import os, sys import tox._config -from tox._venv import * +from tox._venv import * # noqa +from tox.interpreters import NoInterpreterInfo #def test_global_virtualenv(capfd): # v = VirtualEnv() @@ -34,6 +35,12 @@ def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): monkeypatch.setattr(venv.envconfig, 'basepython', 'notexistingpython') py.test.raises(tox.exception.InterpreterNotFound, venv.getsupportedinterpreter) + monkeypatch.undo() + # check that we properly report when no version_info is present + info = NoInterpreterInfo(name=venv.name) + info.executable = "something" + monkeypatch.setattr(config.interpreters, "get_info", lambda *args: info) + pytest.raises(tox.exception.InvocationError, venv.getsupportedinterpreter) def test_create(monkeypatch, mocksession, newconfig): @@ -277,7 +284,7 @@ def test_install_command_not_installed(newmocksession, monkeypatch): venv = mocksession.getenv('python') venv.test() mocksession.report.expect("warning", "*test command found but not*") - assert venv.status == "commands failed" + assert venv.status == 0 def test_install_command_whitelisted(newmocksession, monkeypatch): mocksession = newmocksession(['--recreate'], """ @@ -295,7 +302,7 @@ def test_install_command_whitelisted(newmocksession, monkeypatch): assert venv.status == "commands failed" @pytest.mark.skipif("not sys.platform.startswith('linux')") -def test_install_command_not_installed(newmocksession): +def test_install_command_not_installed_bash(newmocksession): mocksession = newmocksession(['--recreate'], """ [testenv] commands= @@ -387,7 +394,7 @@ class TestCreationConfig: [testenv] deps={distshare}/xyz-* """) - xyz = config.distshare.ensure("xyz-1.2.0.zip") + config.distshare.ensure("xyz-1.2.0.zip") xyz2 = config.distshare.ensure("xyz-1.2.1.zip") envconfig = config.envconfigs['python'] venv = VirtualEnv(envconfig, session=mocksession) @@ -485,9 +492,11 @@ class TestVenvTest: py.test.raises(ZeroDivisionError, "venv._pcall([1,2,3])") monkeypatch.setenv("PIP_RESPECT_VIRTUALENV", "1") monkeypatch.setenv("PIP_REQUIRE_VIRTUALENV", "1") + monkeypatch.setenv("__PYVENV_LAUNCHER__", "1") py.test.raises(ZeroDivisionError, "venv.run_install_command(['qwe'])") assert 'PIP_RESPECT_VIRTUALENV' not in os.environ assert 'PIP_REQUIRE_VIRTUALENV' not in os.environ + assert '__PYVENV_LAUNCHER__' not in os.environ def test_setenv_added_to_pcall(tmpdir, mocksession, newconfig): pkg = tmpdir.ensure("package.tar.gz") @@ -507,11 +516,11 @@ def test_setenv_added_to_pcall(tmpdir, mocksession, newconfig): l = mocksession._pcalls assert len(l) == 2 for x in l: - args = x.args env = x.env assert env is not None assert 'ENV_VAR' in env assert env['ENV_VAR'] == 'value' + assert env['VIRTUAL_ENV'] == str(venv.path) for e in os.environ: assert e in env @@ -553,8 +562,6 @@ def test_run_install_command(newmocksession): assert 'install' in l[0].args env = l[0].env assert env is not None - assert 'PYTHONIOENCODING' in env - assert env['PYTHONIOENCODING'] == 'utf_8' def test_run_custom_install_command(newmocksession): mocksession = newmocksession([], """ @@ -585,6 +592,7 @@ def test_command_relative_issue26(newmocksession, tmpdir, monkeypatch): mocksession.report.not_expect("warning", "*test command found but not*") monkeypatch.setenv("PATH", str(tmpdir)) x4 = venv.getcommandpath("x", cwd=tmpdir) + assert x4.endswith(os.sep + 'x') mocksession.report.expect("warning", "*test command found but not*") def test_sethome_only_on_option(newmocksession, monkeypatch): diff --git a/tests/test_z_cmdline.py b/tests/test_z_cmdline.py index 343c142..00188ac 100644 --- a/tests/test_z_cmdline.py +++ b/tests/test_z_cmdline.py @@ -1,7 +1,6 @@ import tox import py import pytest -import sys from tox._pytestplugin import ReportExpectMock try: import json @@ -129,7 +128,6 @@ class TestSession: }) config = parseconfig([]) session = Session(config) - envlist = ['hello', 'world'] envs = session.venvlist assert len(envs) == 2 env1, env2 = envs @@ -193,6 +191,19 @@ def test_minversion(cmd, initproj): ]) assert result.ret +def test_run_custom_install_command_error(cmd, initproj): + initproj("interp123-0.5", filedefs={ + 'tox.ini': ''' + [testenv] + install_command=./tox.ini {opts} {packages} + ''' + }) + result = cmd.run("tox") + result.stdout.fnmatch_lines([ + "ERROR: invocation failed (errno *), args: ['*/tox.ini*", + ]) + assert result.ret + def test_unknown_interpreter_and_env(cmd, initproj): initproj("interp123-0.5", filedefs={ 'tests': {'test_hello.py': "def test_hello(): pass"}, @@ -231,6 +242,22 @@ def test_unknown_interpreter(cmd, initproj): "*ERROR*InterpreterNotFound*xyz_unknown_interpreter*", ]) +def test_skip_unknown_interpreter(cmd, initproj): + initproj("interp123-0.5", filedefs={ + 'tests': {'test_hello.py': "def test_hello(): pass"}, + 'tox.ini': ''' + [testenv:python] + basepython=xyz_unknown_interpreter + [testenv] + changedir=tests + ''' + }) + result = cmd.run("tox", "--skip-missing-interpreters") + assert not result.ret + result.stdout.fnmatch_lines([ + "*SKIPPED*InterpreterNotFound*xyz_unknown_interpreter*", + ]) + def test_unknown_dep(cmd, initproj): initproj("dep123-0.7", filedefs={ 'tests': {'test_hello.py': "def test_hello(): pass"}, @@ -567,7 +594,7 @@ def test_sdistonly(initproj, cmd): result.stdout.fnmatch_lines([ "*sdist-make*setup.py*", ]) - assert "virtualenv" not in result.stdout.str() + assert "-mvirtualenv" not in result.stdout.str() def test_separate_sdist_no_sdistfile(cmd, initproj): distshare = cmd.tmpdir.join("distshare") @@ -582,6 +609,7 @@ def test_separate_sdist_no_sdistfile(cmd, initproj): l = distshare.listdir() assert len(l) == 1 sdistfile = l[0] + assert 'pkg123-0.7.zip' in str(sdistfile) def test_separate_sdist(cmd, initproj): distshare = cmd.tmpdir.join("distshare") @@ -611,7 +639,6 @@ def test_sdist_latest(tmpdir, newconfig): distshare=%s sdistsrc={distshare}/pkg123-* """ % distshare) - p0 = distshare.ensure("pkg123-1.3.5.zip") p = distshare.ensure("pkg123-1.4.5.zip") distshare.ensure("pkg123-1.4.5a1.zip") session = Session(config) @@ -1,5 +1,5 @@ [tox] -envlist=py27,py33,py26,py32,pypy +envlist=py27,py26,py34,py33,py32,pypy,flakes [testenv:X] commands=echo {posargs} @@ -18,6 +18,10 @@ commands= --junitxml={envlogdir}/junit-{envname}.xml \ check_sphinx.py {posargs} +[testenv:flakes] +deps = pytest-flakes>=0.2 +commands = py.test --flakes -m flakes tox tests + [testenv:py25] setenv= PIP_INSECURE=1 diff --git a/tox/__init__.py b/tox/__init__.py index 8f0acb1..47b85e4 100644 --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '1.7.1' +__version__ = '1.8.1.dev1' class exception: class Error(Exception): @@ -20,4 +20,4 @@ class exception: class MissingDependency(Error): """ a dependency could not be found or determined. """ -from tox._cmdline import main as cmdline +from tox._cmdline import main as cmdline # noqa diff --git a/tox/_cmdline.py b/tox/_cmdline.py index 7c6cd69..0df2f17 100644 --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -79,7 +79,6 @@ class Action(object): return f def popen(self, args, cwd=None, env=None, redirect=True, returnout=False): - logged_command = "%s$ %s" %(cwd, " ".join(map(str, args))) f = outpath = None resultjson = self.session.config.option.resultjson if resultjson or redirect: @@ -93,8 +92,13 @@ class Action(object): if cwd is None: # XXX cwd = self.session.config.cwd cwd = py.path.local() - popen = self._popen(args, cwd, env=env, - stdout=f, stderr=STDOUT) + try: + popen = self._popen(args, cwd, env=env, + stdout=f, stderr=STDOUT) + except OSError as e: + self.report.error("invocation failed (errno %d), args: %s, cwd: %s" % + (e.errno, args, cwd)) + raise popen.outpath = outpath popen.args = [str(x) for x in args] popen.cwd = cwd @@ -114,8 +118,8 @@ class Action(object): if ret: invoked = " ".join(map(str, popen.args)) if outpath: - self.report.error("invocation failed, logfile: %s" % - outpath) + self.report.error("invocation failed (exit code %d), logfile: %s" % + (ret, outpath)) out = outpath.read() self.report.error(out) if hasattr(self, "commandlog"): @@ -223,6 +227,9 @@ class Reporter(object): def error(self, msg): self.logline("ERROR: " + msg, red=True) + def skip(self, msg): + self.logline("SKIPPED:" + msg, yellow=True) + def logline(self, msg, **opts): self._reportedlines.append(msg) self.tw.line("%s" % msg, **opts) @@ -411,7 +418,8 @@ class Session: sdist_path = self._makesdist() except tox.exception.InvocationError: v = sys.exc_info()[1] - self.report.error("FAIL could not package project") + self.report.error("FAIL could not package project - v = %r" % + v) return sdistfile = self.config.distshare.join(sdist_path.basename) if sdistfile != sdist_path: @@ -461,7 +469,14 @@ class Session: retcode = 0 for venv in self.venvlist: status = venv.status - if status and status != "skipped tests": + if isinstance(status, tox.exception.InterpreterNotFound): + msg = " %s: %s" %(venv.envconfig.envname, str(status)) + if self.config.option.skip_missing_interpreters: + self.report.skip(msg) + else: + retcode = 1 + self.report.error(msg) + elif status and status != "skipped tests": msg = " %s: %s" %(venv.envconfig.envname, str(status)) self.report.error(msg) retcode = 1 diff --git a/tox/_config.py b/tox/_config.py index 4d54924..f0ad87e 100644 --- a/tox/_config.py +++ b/tox/_config.py @@ -1,14 +1,12 @@ import argparse -import distutils.sysconfig import os import random import sys import re import shlex import string -import subprocess -import textwrap import pkg_resources +import itertools from tox.interpreters import Interpreters @@ -18,13 +16,10 @@ import tox iswin32 = sys.platform == "win32" -defaultenvs = {'jython': 'jython', 'pypy': 'pypy'} -for _name in "py,py24,py25,py26,py27,py30,py31,py32,py33,py34".split(","): - if _name == "py": - basepython = sys.executable - else: - basepython = "python" + ".".join(_name[2:4]) - defaultenvs[_name] = basepython +default_factors = {'jython': 'jython', 'pypy': 'pypy', 'pypy3': 'pypy3', + 'py': sys.executable} +for version in '24,25,26,27,30,31,32,33,34'.split(','): + default_factors['py' + version] = 'python%s.%s' % tuple(version) def parseconfig(args=None, pkg=None): if args is None: @@ -123,7 +118,8 @@ def prepare_parse(pkgname): parser.add_argument("--hashseed", action="store", metavar="SEED", default=None, help="set PYTHONHASHSEED to SEED before running commands. " - "Defaults to a random integer in the range 1 to 4294967295. " + "Defaults to a random integer in the range [1, 4294967295] " + "([1, 1024] on Windows). " "Passing 'noset' suppresses this behavior.") parser.add_argument("--force-dep", action="append", metavar="REQ", default=None, @@ -132,6 +128,8 @@ def prepare_parse(pkgname): "'pytest<2.7' or 'django>=1.6'.") parser.add_argument("--sitepackages", action="store_true", help="override sitepackages setting to True in all envs") + parser.add_argument("--skip-missing-interpreters", action="store_true", + help="don't fail tests for missing interpreters") parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") @@ -186,6 +184,9 @@ class VenvConfig: info = self.config.interpreters.get_info(self.basepython) if not info.executable: raise tox.exception.InterpreterNotFound(self.basepython) + if not info.version_info: + raise tox.exception.InvocationError( + 'Failed to get version_info for %s: %s' % (info.name, info.err)) if info.version_info < (2,6): raise tox.exception.UnsupportedInterpreter( "python2.5 is not supported anymore, sorry") @@ -200,12 +201,15 @@ def get_homedir(): return None def make_hashseed(): - return str(random.randint(1, 4294967295)) + max_seed = 4294967295 + if sys.platform == 'win32': + max_seed = 1024 + return str(random.randint(1, max_seed)) class parseini: def __init__(self, config, inipath): config.toxinipath = inipath - config.toxinidir = toxinidir = config.toxinipath.dirpath() + config.toxinidir = config.toxinipath.dirpath() self._cfg = py.iniconfig.IniConfig(config.toxinipath) config._cfg = self._cfg @@ -236,10 +240,14 @@ class parseini: "{toxinidir}/.tox") config.minversion = reader.getdefault(toxsection, "minversion", None) + if not config.option.skip_missing_interpreters: + config.option.skip_missing_interpreters = \ + reader.getbool(toxsection, "skip_missing_interpreters", False) + # determine indexserver dictionary config.indexserver = {'default': IndexServerConfig('default')} prefix = "indexserver" - for line in reader.getlist(toxsection, "indexserver"): + for line in reader.getlist(toxsection, prefix): name, url = map(lambda x: x.strip(), line.split("=", 1)) config.indexserver[name] = IndexServerConfig(name, url) @@ -273,22 +281,19 @@ class parseini: config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None) config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") - for sectionwrapper in self._cfg: - section = sectionwrapper.name - if section.startswith(testenvprefix): - name = section[len(testenvprefix):] - envconfig = self._makeenvconfig(name, section, reader._subs, - config) - config.envconfigs[name] = envconfig - if not config.envconfigs: - config.envconfigs['python'] = \ - self._makeenvconfig("python", "_xz_9", reader._subs, config) - config.envlist = self._getenvlist(reader, toxsection) - for name in config.envlist: - if name not in config.envconfigs: - if name in defaultenvs: - config.envconfigs[name] = \ - self._makeenvconfig(name, "_xz_9", reader._subs, config) + + config.envlist, all_envs = self._getenvdata(reader, toxsection) + + # configure testenvs + known_factors = self._list_section_factors("testenv") + known_factors.update(default_factors) + known_factors.add("python") + for name in all_envs: + section = testenvprefix + name + factors = set(name.split('-')) + if section in self._cfg or factors <= known_factors: + config.envconfigs[name] = \ + self._makeenvconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs and config.envconfigs[name].develop @@ -296,10 +301,20 @@ class parseini: config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop) + def _list_section_factors(self, section): + factors = set() + if section in self._cfg: + for _, value in self._cfg[section].items(): + exprs = re.findall(r'^([\w{}\.,-]+)\:\s+', value, re.M) + factors.update(*mapcat(_split_factor_expr, exprs)) + return factors + def _makeenvconfig(self, name, section, subs, config): vc = VenvConfig(envname=name) vc.config = config - reader = IniReader(self._cfg, fallbacksections=["testenv"]) + factors = set(name.split('-')) + reader = IniReader(self._cfg, fallbacksections=["testenv"], + factors=factors) reader.addsubstitutions(**subs) vc.develop = not config.option.installpkg and \ reader.getbool(section, "usedevelop", config.option.develop) @@ -308,10 +323,8 @@ class parseini: if reader.getdefault(section, "python", None): raise tox.exception.ConfigError( "'python=' key was renamed to 'basepython='") - if name in defaultenvs: - bp = defaultenvs[name] - else: - bp = sys.executable + bp = next((default_factors[f] for f in factors if f in default_factors), + sys.executable) vc.basepython = reader.getdefault(section, "basepython", bp) vc._basepython_info = config.interpreters.get_info(vc.basepython) reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname, @@ -379,20 +392,25 @@ class parseini: "'install_command' must contain '{packages}' substitution") return vc - def _getenvlist(self, reader, toxsection): - env = self.config.option.env - if not env: - env = os.environ.get("TOXENV", None) - if not env: - envlist = reader.getlist(toxsection, "envlist", sep=",") - if not envlist: - envlist = self.config.envconfigs.keys() - return envlist - envlist = _split_env(env) - if "ALL" in envlist: - envlist = list(self.config.envconfigs) - envlist.sort() - return envlist + def _getenvdata(self, reader, toxsection): + envstr = self.config.option.env \ + or os.environ.get("TOXENV") \ + or reader.getdefault(toxsection, "envlist", replace=False) \ + or [] + envlist = _split_env(envstr) + + # collect section envs + all_envs = set(envlist) - set(["ALL"]) + for section in self._cfg: + if section.name.startswith(testenvprefix): + all_envs.add(section.name[len(testenvprefix):]) + if not all_envs: + all_envs.add("python") + + if not envlist or "ALL" in envlist: + envlist = sorted(all_envs) + + return envlist, all_envs def _replace_forced_dep(self, name, config): """ @@ -420,17 +438,32 @@ class parseini: dep2_name = pkg_resources.Requirement.parse(dep2).project_name return dep1_name == dep2_name + def _split_env(env): """if handed a list, action="append" was used for -e """ - envlist = [] if not isinstance(env, list): env = [env] - for to_split in env: - for single_env in to_split.split(","): - # "remove True or", if not allowing multiple same runs, update tests - if True or single_env not in envlist: - envlist.append(single_env) - return envlist + return mapcat(_expand_envstr, env) + +def _split_factor_expr(expr): + partial_envs = _expand_envstr(expr) + return [set(e.split('-')) for e in partial_envs] + +def _expand_envstr(envstr): + # split by commas not in groups + tokens = re.split(r'(\{[^}]+\})|,', envstr) + envlist = [''.join(g).strip() + for k, g in itertools.groupby(tokens, key=bool) if k] + + def expand(env): + tokens = re.split(r'\{([^}]+)\}', env) + parts = [token.split(',') for token in tokens] + return [''.join(variant) for variant in itertools.product(*parts)] + + return mapcat(expand, envlist) + +def mapcat(f, seq): + return list(itertools.chain.from_iterable(map(f, seq))) class DepConfig: def __init__(self, name, indexserver=None): @@ -461,9 +494,10 @@ RE_ITEM_REF = re.compile( class IniReader: - def __init__(self, cfgparser, fallbacksections=None): + def __init__(self, cfgparser, fallbacksections=None, factors=()): self._cfg = cfgparser self.fallbacksections = fallbacksections or [] + self.factors = factors self._subs = {} self._subststack = [] @@ -492,6 +526,8 @@ class IniReader: value = {} for line in s.split(sep): + if not line.strip(): + continue name, rest = line.split('=', 1) value[name.strip()] = rest.strip() @@ -527,30 +563,35 @@ class IniReader: def _processcommand(self, command): posargs = getattr(self, "posargs", None) - # special treat posargs which might contain multiple arguments - # in their defaults + # Iterate through each word of the command substituting as + # appropriate to construct the new command string. This + # string is then broken up into exec argv components using + # shlex. newcommand = "" for word in CommandParser(command).words(): - if word.startswith("{posargs:") and word.endswith("}"): + if word == "{posargs}" or word == "[]": + if posargs: + newcommand += " ".join(posargs) + continue + elif word.startswith("{posargs:") and word.endswith("}"): if posargs: - word = "{posargs}" + newcommand += " ".join(posargs) + continue else: word = word[9:-1] - newcommand += word - - # now we can properly parse the command - argv = [] - for arg in shlex.split(newcommand): - if arg in ('[]', "{posargs}"): - if posargs: - argv.extend(posargs) - continue new_arg = "" - for word in CommandParser(arg).words(): - new_word = self._replace(word) - new_word = self._replace(new_word) - new_arg += new_word - argv.append(new_arg) + new_word = self._replace(word) + new_word = self._replace(new_word) + new_arg += new_word + newcommand += new_arg + + # Construct shlex object that will not escape any values, + # use all values as is in argv. + shlexer = shlex.shlex(newcommand, posix=True) + shlexer.whitespace_split = True + shlexer.escape = '' + shlexer.commenters = '' + argv = list(shlexer) return argv def getargv(self, section, name, default=None, replace=True): @@ -560,9 +601,12 @@ class IniReader: def getbool(self, section, name, default=None): s = self.getdefault(section, name, default) + if not s: + s = default if s is None: raise KeyError("no config value [%s] %s found" % ( section, name)) + if not isinstance(s, bool): if s.lower() == "true": s = True @@ -574,18 +618,19 @@ class IniReader: return s def getdefault(self, section, name, default=None, replace=True): - try: - x = self._cfg[section][name] - except KeyError: - for fallbacksection in self.fallbacksections: - try: - x = self._cfg[fallbacksection][name] - except KeyError: - pass - else: - break - else: - x = default + x = None + for s in [section] + self.fallbacksections: + try: + x = self._cfg[s][name] + break + except KeyError: + continue + + if x is None: + x = default + else: + x = self._apply_factors(x) + if replace and x and hasattr(x, 'replace'): self._subststack.append((section, name)) try: @@ -595,18 +640,39 @@ class IniReader: #print "getdefault", section, name, "returned", repr(x) return x + def _apply_factors(self, s): + def factor_line(line): + m = re.search(r'^([\w{}\.,-]+)\:\s+(.+)', line) + if not m: + return line + + expr, line = m.groups() + if any(fs <= self.factors for fs in _split_factor_expr(expr)): + return line + + lines = s.strip().splitlines() + return '\n'.join(filter(None, map(factor_line, lines))) + def _replace_env(self, match): - envkey = match.group('substitution_value') - if not envkey: + match_value = match.group('substitution_value') + if not match_value: raise tox.exception.ConfigError( 'env: requires an environment variable name') - if not envkey in os.environ: + default = None + envkey_split = match_value.split(':', 1) + + if len(envkey_split) is 2: + envkey, default = envkey_split + else: + envkey = match_value + + if not envkey in os.environ and default is None: raise tox.exception.ConfigError( "substitution env:%r: unkown environment variable %r" % (envkey, envkey)) - return os.environ[envkey] + return os.environ.get(envkey, default) def _substitute_from_other_section(self, key): if key.startswith("[") and "]" in key: diff --git a/tox/_quickstart.py b/tox/_quickstart.py index f66f1e2..098eb61 100644 --- a/tox/_quickstart.py +++ b/tox/_quickstart.py @@ -56,7 +56,7 @@ except NameError: term_input = input -all_envs = ['py26', 'py27', 'py32', 'py33', 'pypy', 'jython'] +all_envs = ['py26', 'py27', 'py32', 'py33', 'py34', 'pypy', 'jython'] PROMPT_PREFIX = '> ' diff --git a/tox/_venv.py b/tox/_venv.py index d0f2e64..0c56cc0 100644 --- a/tox/_venv.py +++ b/tox/_venv.py @@ -1,6 +1,6 @@ from __future__ import with_statement -import sys, os, re -import subprocess +import sys, os +import codecs import py import tox from tox._config import DepConfig @@ -121,8 +121,6 @@ class VirtualEnv(object): """ if action is None: action = self.session.newaction(self, "update") - report = self.session.report - name = self.envconfig.envname rconfig = CreationConfig.readconfig(self.path_config) if not self.envconfig.recreate and rconfig and \ rconfig.matches(self._getliveconfig()): @@ -142,7 +140,8 @@ class VirtualEnv(object): self.install_deps(action) except tox.exception.InvocationError: v = sys.exc_info()[1] - return "could not install deps %s" %(self.envconfig.deps,) + return "could not install deps %s; v = %r" % ( + self.envconfig.deps, v) def _getliveconfig(self): python = self.envconfig._basepython_info.executable @@ -273,16 +272,19 @@ class VirtualEnv(object): if '{opts}' in argv: i = argv.index('{opts}') argv[i:i+1] = list(options) - for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV'): + for x in ('PIP_RESPECT_VIRTUALENV', 'PIP_REQUIRE_VIRTUALENV', + '__PYVENV_LAUNCHER__'): try: del os.environ[x] except KeyError: pass - env = dict(PYTHONIOENCODING='utf_8') - if extraenv is not None: - env.update(extraenv) + old_stdout = sys.stdout + sys.stdout = codecs.getwriter('utf8')(sys.stdout) + if extraenv is None: + extraenv = {} self._pcall(argv, cwd=self.envconfig.config.toxinidir, - extraenv=env, action=action) + extraenv=extraenv, action=action) + sys.stdout = old_stdout def _install(self, deps, extraopts=None, action=None): if not deps: @@ -322,6 +324,7 @@ class VirtualEnv(object): setenv = self.envconfig.setenv if setenv: env.update(setenv) + env['VIRTUAL_ENV'] = str(self.path) env.update(extraenv) return env diff --git a/tox/_verlib.py b/tox/_verlib.py index 1df3645..a234176 100644 --- a/tox/_verlib.py +++ b/tox/_verlib.py @@ -8,7 +8,6 @@ licensed under the PSF license (i guess) """ -import sys import re class IrrationalVersionError(Exception): diff --git a/tox/interpreters.py b/tox/interpreters.py index e225fcc..75318c5 100644 --- a/tox/interpreters.py +++ b/tox/interpreters.py @@ -1,8 +1,6 @@ import sys -import os import py import re -import subprocess import inspect class Interpreters: @@ -161,7 +159,7 @@ else: # The standard executables can be found as a last resort via the # Python launcher py.exe if m: - locate_via_py(*m.groups()) + return locate_via_py(*m.groups()) def pyinfo(): import sys |