diff options
-rwxr-xr-x | CHANGELOG | 7 | ||||
-rw-r--r-- | doc/example/result.txt | 44 | ||||
-rw-r--r-- | setup.py | 4 | ||||
-rw-r--r-- | tests/test_result.py | 51 | ||||
-rw-r--r-- | tests/test_venv.py | 25 | ||||
-rw-r--r-- | tests/test_z_cmdline.py | 132 | ||||
-rw-r--r-- | tox.ini | 9 | ||||
-rw-r--r-- | tox/__init__.py | 2 | ||||
-rw-r--r-- | tox/_cmdline.py | 32 | ||||
-rw-r--r-- | tox/_config.py | 16 | ||||
-rw-r--r-- | tox/_pytestplugin.py | 3 | ||||
-rw-r--r-- | tox/_venv.py | 1 | ||||
-rw-r--r-- | tox/result.py | 75 | ||||
-rw-r--r-- | toxbootstrap.py | 2 |
14 files changed, 325 insertions, 78 deletions
@@ -1,12 +1,17 @@ -1.5.1.dev +1.6.0.dev ----------------- +- add --result-json option to write out detailed per-venv information + into a json report file to be used by upstream tools. + - add new config options ``usedevelop`` and ``skipsdist`` as well as a command line option ``--develop`` to install the package-under-test in develop mode. thanks Monty Tailor for the PR. - always unset PYTHONDONTWRITEBYTE because newer setuptools doesn't like it +- if a HOMEDIR cannot be determined, use the toxinidir. + 1.5.0 ----------------- diff --git a/doc/example/result.txt b/doc/example/result.txt new file mode 100644 index 0000000..b3b95b4 --- /dev/null +++ b/doc/example/result.txt @@ -0,0 +1,44 @@ + +Writing a json result file +-------------------------------------------------------- + +.. versionadded: 1.6 + +You can instruct tox to write a json-report file via:: + + tox --result-json=PATH + +This will create a json-formatted result file using this schema:: + + { + "testenvs": { + "py27": { + "python": { + "executable": "/home/hpk/p/tox/.tox/py27/bin/python", + "version": "2.7.3 (default, Aug 1 2012, 05:14:39) \n[GCC 4.6.3]", + "version_info": [ 2, 7, 3, "final", 0 ] + }, + "test": [ + { + "output": "...", + "command": [ + "/home/hpk/p/tox/.tox/py27/bin/py.test", + "--instafail", + "--junitxml=/home/hpk/p/tox/.tox/py27/log/junit-py27.xml", + "tests/test_config.py" + ], + "retcode": "0" + } + ], + "setup": [] + } + }, + "platform": "linux2", + "installpkg": { + "basename": "tox-1.6.0.dev1.zip", + "sha256": "b6982dde5789a167c4c35af0d34ef72176d0575955f5331ad04aee9f23af4326", + "md5": "27ead99fd7fa39ee7614cede6bf175a6" + }, + "toxversion": "1.6.0.dev1", + "reportversion": "1" + } @@ -21,12 +21,14 @@ def main(): install_requires = ['virtualenv>=1.9.1', 'py>=1.4.15', ] if version < (2, 7) or (3, 0) <= version <= (3, 1): install_requires += ['argparse'] + if version < (2,6): + install_requires += ["simplejson"] setup( name='tox', description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='1.5.1.dev2', + version='1.6.0.dev2', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff --git a/tests/test_result.py b/tests/test_result.py new file mode 100644 index 0000000..6df834a --- /dev/null +++ b/tests/test_result.py @@ -0,0 +1,51 @@ +import sys +import py +from tox.result import ResultLog +import tox +import pytest + +@pytest.fixture +def pkg(tmpdir): + p = tmpdir.join("hello-1.0.tar.gz") + p.write("whatever") + return p + +def test_set_header(pkg): + replog = ResultLog() + d = replog.dict + replog.set_header(installpkg=pkg) + assert replog.dict == d + assert replog.dict["reportversion"] == "1" + assert replog.dict["toxversion"] == tox.__version__ + assert replog.dict["platform"] == sys.platform + assert replog.dict["host"] == py.std.socket.getfqdn() + assert replog.dict["installpkg"] == {"basename": "hello-1.0.tar.gz", + "md5": pkg.computehash("md5"), + "sha256": pkg.computehash("sha256")} + data = replog.dumps_json() + replog2 = ResultLog.loads_json(data) + assert replog2.dict == replog.dict + +def test_addenv_setpython(pkg): + replog = ResultLog() + replog.set_header(installpkg=pkg) + envlog = replog.get_envlog("py26") + envlog.set_python_info(py.path.local(sys.executable)) + assert envlog.dict["python"]["version_info"] == list(sys.version_info) + assert envlog.dict["python"]["version"] == sys.version + assert envlog.dict["python"]["executable"] == sys.executable + +def test_get_commandlog(pkg): + replog = ResultLog() + replog.set_header(installpkg=pkg) + envlog = replog.get_envlog("py26") + assert "setup" not in envlog.dict + setuplog = envlog.get_commandlog("setup") + setuplog.add_command(["virtualenv", "..."], "venv created", 0) + assert setuplog.list == [{"command": ["virtualenv", "..."], + "output": "venv created", + "retcode": "0"}] + assert envlog.dict["setup"] + setuplog2 = replog.get_envlog("py26").get_commandlog("setup") + assert setuplog2.list == setuplog.list + diff --git a/tests/test_venv.py b/tests/test_venv.py index 8c5396a..badc3b8 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -264,14 +264,15 @@ def test_installpkg_indexserver(newmocksession, tmpdir): args = " ".join(l[0].args) assert "-i ABC" in args -def test_install_recreate(newmocksession): +def test_install_recreate(newmocksession, tmpdir): + pkg = tmpdir.ensure("package.tar.gz") mocksession = newmocksession(['--recreate'], """ [testenv] deps=xyz """) venv = mocksession.getenv('python') venv.update() - mocksession.installpkg(venv, "xz") + mocksession.installpkg(venv, pkg) mocksession.report.expect("verbosity0", "*create*") venv.update() mocksession.report.expect("verbosity0", "*recreate*") @@ -426,14 +427,15 @@ class TestCreationConfig: assert path == xyz2 assert md5 == path.computehash() - def test_python_recreation(self, newconfig, mocksession): + def test_python_recreation(self, tmpdir, newconfig, mocksession): + pkg = tmpdir.ensure("package.tar.gz") config = newconfig([], "") envconfig = config.envconfigs['python'] venv = VirtualEnv(envconfig, session=mocksession) cconfig = venv._getliveconfig() venv.update() assert not venv.path_config.check() - mocksession.installpkg(venv, "sdist.zip") + mocksession.installpkg(venv, pkg) assert venv.path_config.check() assert mocksession._pcalls args1 = map(str, mocksession._pcalls[0].args) @@ -519,7 +521,8 @@ class TestVenvTest: assert 'PIP_RESPECT_VIRTUALENV' not in os.environ assert 'PIP_REQUIRE_VIRTUALENV' not in os.environ -def test_setenv_added_to_pcall(mocksession, newconfig): +def test_setenv_added_to_pcall(tmpdir, mocksession, newconfig): + pkg = tmpdir.ensure("package.tar.gz") config = newconfig([], """ [testenv:python] commands=python -V @@ -530,7 +533,7 @@ def test_setenv_added_to_pcall(mocksession, newconfig): venv = VirtualEnv(config.envconfigs['python'], session=mocksession) # import pdb; pdb.set_trace() - mocksession.installpkg(venv, "xyz") + mocksession.installpkg(venv, pkg) venv.test() l = mocksession._pcalls @@ -545,21 +548,23 @@ def test_setenv_added_to_pcall(mocksession, newconfig): for e in os.environ: assert e in env -def test_installpkg_no_upgrade(newmocksession): +def test_installpkg_no_upgrade(tmpdir, newmocksession): + pkg = tmpdir.ensure("package.tar.gz") mocksession = newmocksession([], "") venv = mocksession.getenv('python') venv.just_created = True venv.envconfig.envdir.ensure(dir=1) - mocksession.installpkg(venv, "whatever") + mocksession.installpkg(venv, pkg) l = mocksession._pcalls assert len(l) == 1 assert '-U' not in l[0].args -def test_installpkg_upgrade(newmocksession): +def test_installpkg_upgrade(newmocksession, tmpdir): + pkg = tmpdir.ensure("package.tar.gz") mocksession = newmocksession([], "") venv = mocksession.getenv('python') assert not hasattr(venv, 'just_created') - mocksession.installpkg(venv, "whatever") + mocksession.installpkg(venv, pkg) l = mocksession._pcalls assert len(l) == 1 assert '-U' in l[0].args diff --git a/tests/test_z_cmdline.py b/tests/test_z_cmdline.py index c9596ec..8cc0e85 100644 --- a/tests/test_z_cmdline.py +++ b/tests/test_z_cmdline.py @@ -263,7 +263,7 @@ def test_skip_sdist(cmd, initproj): [tox] skipsdist=True [testenv] - commands=echo done + commands=python -c "print('done')" ''' }) result = cmd.run("tox", ) @@ -308,55 +308,64 @@ def test_package_install_fails(cmd, initproj): "*InvocationError*", ]) -def test_test_simple(cmd, initproj): - initproj("example123-0.5", filedefs={ - 'tests': {'test_hello.py': """ - def test_hello(pytestconfig): - pass - """, - }, - 'tox.ini': ''' - [testenv] - changedir=tests - commands= - py.test --basetemp={envtmpdir} --junitxml=junit-{envname}.xml [] - deps=pytest - ''' - }) - result = cmd.run("tox") - assert not result.ret - result.stdout.fnmatch_lines([ - "*junit-python.xml*", - "*1 passed*", - ]) - result = cmd.run("tox", "-epython", ) - assert not result.ret - result.stdout.fnmatch_lines([ - "*1 passed*", - "*summary*", - "*python: commands succeeded" - ]) - # see that things work with a different CWD - old = cmd.tmpdir.chdir() - result = cmd.run("tox", "-c", "example123/tox.ini") - assert not result.ret - result.stdout.fnmatch_lines([ - "*1 passed*", - "*summary*", - "*python: commands succeeded" - ]) - old.chdir() - # see that tests can also fail and retcode is correct - testfile = py.path.local("tests").join("test_hello.py") - assert testfile.check() - testfile.write("def test_fail(): assert 0") - result = cmd.run("tox", ) - assert result.ret - result.stdout.fnmatch_lines([ - "*1 failed*", - "*summary*", - "*python: *failed*", - ]) +class TestToxRun: + @pytest.fixture + def example123(self, initproj): + initproj("example123-0.5", filedefs={ + 'tests': {'test_hello.py': """ + def test_hello(pytestconfig): + pass + """, + }, + 'tox.ini': ''' + [testenv] + changedir=tests + commands= py.test --basetemp={envtmpdir} --junitxml=junit-{envname}.xml + deps=pytest + ''' + }) + + def test_toxuone_env(self, cmd, example123): + result = cmd.run("tox") + assert not result.ret + result.stdout.fnmatch_lines([ + "*junit-python.xml*", + "*1 passed*", + ]) + result = cmd.run("tox", "-epython", ) + assert not result.ret + result.stdout.fnmatch_lines([ + "*1 passed*", + "*summary*", + "*python: commands succeeded" + ]) + + def test_different_config_cwd(self, cmd, example123, monkeypatch): + # see that things work with a different CWD + monkeypatch.chdir(cmd.tmpdir) + result = cmd.run("tox", "-c", "example123/tox.ini") + assert not result.ret + result.stdout.fnmatch_lines([ + "*1 passed*", + "*summary*", + "*python: commands succeeded" + ]) + + def test_json(self, cmd, example123): + # see that tests can also fail and retcode is correct + testfile = py.path.local("tests").join("test_hello.py") + assert testfile.check() + testfile.write("def test_fail(): assert 0") + jsonpath = cmd.tmpdir.join("res.json") + result = cmd.run("tox", "--result-json", jsonpath) + assert result.ret == 1 + data = py.std.json.load(jsonpath.open("r")) + verify_json_report_format(data) + result.stdout.fnmatch_lines([ + "*1 failed*", + "*summary*", + "*python: *failed*", + ]) def test_develop(initproj, cmd): @@ -460,7 +469,7 @@ def test_notest(initproj, cmd): "*py25*reusing*", ]) -def test_env_PYTHONDONTWRITEBYTECODE(initproj, cmd, monkeypatch): +def test_PYC(initproj, cmd, monkeypatch): initproj("example123", filedefs={'tox.ini': ''}) monkeypatch.setenv("PYTHONDOWNWRITEBYTECODE", 1) result = cmd.run("tox", "-v", "--notest") @@ -544,16 +553,33 @@ def test_installpkg(tmpdir, newconfig): sdist_path = session.sdist() assert sdist_path == p -@pytest.mark.xfail("sys.platform == 'win32'", reason="test needs better impl") +#@pytest.mark.xfail("sys.platform == 'win32'", reason="test needs better impl") def test_envsitepackagesdir(cmd, initproj): initproj("pkg512-0.0.5", filedefs={ 'tox.ini': """ [testenv] commands= - echo X:{envsitepackagesdir} + python -c "print('X:{envsitepackagesdir}')" """}) result = cmd.run("tox") assert result.ret == 0 result.stdout.fnmatch_lines(""" X:*site-packages* """) + +def verify_json_report_format(data, testenvs=True): + assert data["reportversion"] == "1" + assert data["toxversion"] == tox.__version__ + if testenvs: + for envname, envdata in data["testenvs"].items(): + for commandtype in ("setup", "test"): + if commandtype not in envdata: + continue + for command in envdata[commandtype]: + assert command["output"] + assert command["retcode"] + pyinfo = envdata["python"] + assert isinstance(pyinfo["version_info"], list) + assert pyinfo["version"] + assert pyinfo["executable"] + @@ -1,15 +1,14 @@ [tox] -envlist=py27,py26,py25,py32,py33,docs,pypy +envlist=py27,py26,py32,py33,docs,pypy [testenv:X] commands=echo {posargs} [testenv] -commands=py.test --instafail --junitxml={envlogdir}/junit-{envname}.xml {posargs} -deps=pytest==2.3.4 - pytest-instafail +commands=py.test --junitxml={envlogdir}/junit-{envname}.xml {posargs} +deps=pytest>=2.3.5 -[testenv:py25] +[testenv:py25] # requires virtualenv-1.9.1 setenvs = PIP_INSECURE=True diff --git a/tox/__init__.py b/tox/__init__.py index 99117b2..1ec4957 100644 --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '1.5.1.dev2' +__version__ = '1.6.0.dev2' class exception: class Error(Exception): diff --git a/tox/_cmdline.py b/tox/_cmdline.py index 3553a20..b205eab 100644 --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -14,6 +14,7 @@ import subprocess from tox._verlib import NormalizedVersion, IrrationalVersionError from tox._venv import VirtualEnv from tox._config import parseconfig +from tox.result import ResultLog from subprocess import STDOUT def now(): @@ -41,6 +42,10 @@ class Action(object): self.venvname = self.venv.name else: self.venvname = "GLOB" + cat = {"runtests": "test", "getenv": "setup"}.get(msg) + if cat: + envlog = session.resultlog.get_envlog(self.venvname) + self.commandlog = envlog.get_commandlog(cat) def __enter__(self): self.report.logaction_start(self) @@ -76,7 +81,8 @@ class Action(object): def popen(self, args, cwd=None, env=None, redirect=True, returnout=False): logged_command = "%s$ %s" %(cwd, " ".join(map(str, args))) f = outpath = None - if redirect: + resultjson = self.session.config.option.resultjson + if resultjson or redirect: f = self._initlogpath(self.id) f.write("actionid=%s\nmsg=%s\ncmdargs=%r\nenv=%s\n" %( self.id, self.msg, args, env)) @@ -89,7 +95,7 @@ class Action(object): cwd = py.path.local() popen = self._popen(args, cwd, env=env, stdout=f, stderr=STDOUT) popen.outpath = outpath - popen.args = args + popen.args = [str(x) for x in args] popen.cwd = cwd popen.action = self self._popenlist.append(popen) @@ -109,11 +115,18 @@ class Action(object): if outpath: self.report.error("invocation failed, logfile: %s" % outpath) - self.report.error(outpath.read()) + out = outpath.read() + self.report.error(out) + if hasattr(self, "commandlog"): + self.commandlog.add_command(popen.args, out, ret) raise tox.exception.InvocationError( "%s (see %s)" %(invoked, outpath), ret) else: raise tox.exception.InvocationError("%r" %(invoked, )) + if not out and outpath: + out = outpath.read() + if hasattr(self, "commandlog"): + self.commandlog.add_command(popen.args, out, ret) return out def _rewriteargs(self, cwd, args): @@ -233,6 +246,7 @@ class Session: def __init__(self, config, popen=subprocess.Popen, Report=Reporter): self.config = config self.popen = popen + self.resultlog = ResultLog() self.report = Report(self) self.make_emptydir(config.logdir) config.logdir.ensure(dir=1) @@ -319,14 +333,19 @@ class Session: action = self.newaction(venv, "getenv", venv.envconfig.envdir) with action: venv.status = 0 + envlog = self.resultlog.get_envlog(venv.name) try: status = venv.update(action=action) except tox.exception.InvocationError: status = sys.exc_info()[1] if status: + commandlog = envlog.get_commandlog("setup") + commandlog.add_command(["setup virtualenv"], str(status), 1) venv.status = status self.report.error(str(status)) return False + commandpath = venv.getcommandpath("python") + envlog.set_python_info(commandpath) return True def finishvenv(self, venv): @@ -346,6 +365,7 @@ class Session: return False def installpkg(self, venv, sdist_path): + self.resultlog.set_header(installpkg=sdist_path) action = self.newaction(venv, "installpkg", sdist_path) with action: try: @@ -425,6 +445,12 @@ class Session: status)) if not retcode: self.report.good(" congratulations :)") + + path = self.config.option.resultjson + if path: + path = py.path.local(path) + path.write(self.resultlog.dumps_json()) + self.report.line("wrote json report at: %s" % path) return retcode def showconfig(self): diff --git a/tox/_config.py b/tox/_config.py index eceacbb..009b92f 100644 --- a/tox/_config.py +++ b/tox/_config.py @@ -90,11 +90,13 @@ def prepare_parse(pkgname): help="skip invoking test commands.") parser.add_argument("--sdistonly", action="store_true", dest="sdistonly", help="only perform the sdist packaging activity.") + parser.add_argument("--installpkg", action="store", default=None, + metavar="PATH", + help="use specified package for installation into venv, instead of " + "creating an sdist.") parser.add_argument("--develop", action="store_true", dest="develop", - help="install package in the venv using setup.py develop using " + help="install package in the venv using 'setup.py develop' via " "'pip -e .'") - parser.add_argument("--installpkg", action="store", default=None, - help="use specified package for installation into venv") parser.add_argument('-i', action="append", dest="indexurl", metavar="URL", help="set indexserver url (if URL is of form name=url set the " @@ -102,6 +104,12 @@ def prepare_parse(pkgname): parser.add_argument("-r", "--recreate", action="store_true", dest="recreate", help="force recreation of virtual environments") + parser.add_argument("--result-json", action="store", + dest="resultjson", metavar="PATH", + help="write a json file with detailed information about " + "all commands and results involved. This will turn off " + "pass-through output from running test commands which is " + "instead captured into the json result file.") parser.add_argument("args", nargs="*", help="additional arguments available to command positional substition") return parser @@ -193,6 +201,8 @@ class parseini: raise ValueError("invalid context") config.homedir = py.path.local._gethomedir() + if config.homedir is None: + config.homedir = config.toxinidir # XXX good idea? reader.addsubstitions(toxinidir=config.toxinidir, homedir=config.homedir) config.toxworkdir = reader.getpath(toxsection, "toxworkdir", diff --git a/tox/_pytestplugin.py b/tox/_pytestplugin.py index 1c0024b..6c90c8d 100644 --- a/tox/_pytestplugin.py +++ b/tox/_pytestplugin.py @@ -8,6 +8,7 @@ import time from tox._config import parseconfig from tox._venv import VirtualEnv from tox._cmdline import Action +from tox.result import ResultLog def pytest_configure(): if 'TOXENV' in os.environ: @@ -118,6 +119,7 @@ def pytest_funcarg__mocksession(request): def __init__(self): self._clearmocks() self.config = request.getfuncargvalue("newconfig")([], "") + self.resultlog = ResultLog() self._actions = [] def getenv(self, name): return VirtualEnv(self.config.envconfigs[name], session=self) @@ -164,6 +166,7 @@ class Cmd: def run(self, *argv): argv = [str(x) for x in argv] + assert py.path.local.sysfind(str(argv[0])), argv[0] p1 = self.tmpdir.join("stdout") p2 = self.tmpdir.join("stderr") print("%s$ %s" % (os.getcwd(), " ".join(argv))) diff --git a/tox/_venv.py b/tox/_venv.py index 044bcf4..b43ab3f 100644 --- a/tox/_venv.py +++ b/tox/_venv.py @@ -358,6 +358,7 @@ class VirtualEnv(object): oldPATH = os.environ['PATH'] bindir = str(self.envconfig.envbindir) os.environ['PATH'] = os.pathsep.join([bindir, oldPATH]) + self.session.report.verbosity2("setting PATH=%s" % os.environ["PATH"]) return oldPATH def getdigest(path): diff --git a/tox/result.py b/tox/result.py new file mode 100644 index 0000000..694138c --- /dev/null +++ b/tox/result.py @@ -0,0 +1,75 @@ +import sys +import py +try: + import json +except ImportError: + import simplejson as json + +class ResultLog: + + def __init__(self, dict=None): + if dict is None: + dict = {} + self.dict = dict + + def set_header(self, installpkg): + from tox import __version__ as toxver + self.dict.update({"reportversion": "1", "toxversion": toxver}) + self.dict["platform"] = sys.platform + self.dict["host"] = py.std.socket.getfqdn() + self.dict["installpkg"] = dict( + md5=installpkg.computehash("md5"), + sha256=installpkg.computehash("sha256"), + basename=installpkg.basename, + ) + + def get_envlog(self, name): + testenvs = self.dict.setdefault("testenvs", {}) + d = testenvs.setdefault(name, {}) + return EnvLog(self, name, d) + + def dumps_json(self): + return json.dumps(self.dict, indent=2) + + @classmethod + def loads_json(cls, data): + return cls(json.loads(data)) + +class EnvLog: + def __init__(self, reportlog, name, dict): + self.reportlog = reportlog + self.name = name + self.dict = dict + + def set_python_info(self, pythonexecutable): + pythonexecutable = py.path.local(pythonexecutable) + out = pythonexecutable.sysexec("-c", + "import sys; " + "print (sys.executable);" + "print (list(sys.version_info)); " + "print (sys.version)") + lines = out.splitlines() + executable = lines.pop(0) + version_info = eval(lines.pop(0)) + version = "\n".join(lines) + self.dict["python"] = dict( + executable=executable, + version_info = version_info, + version = version) + + def get_commandlog(self, name): + l = self.dict.setdefault(name, []) + return CommandLog(self, l) + +class CommandLog: + def __init__(self, envlog, list): + self.envlog = envlog + self.list = list + + def add_command(self, argv, output, retcode): + d = {} + self.list.append(d) + d["command"] = argv + d["output"] = output + d["retcode"] = str(retcode) + return d diff --git a/toxbootstrap.py b/toxbootstrap.py index 514c6db..8f4aebd 100644 --- a/toxbootstrap.py +++ b/toxbootstrap.py @@ -58,7 +58,7 @@ ToDo """ -__version__ = '1.5.1.dev2' +__version__ = '1.6.0.dev2' import sys import os |