From aca9b43f6519aaced355c17e8cf715987915639c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 15 Aug 2013 13:00:39 +0200 Subject: move all interpreter information detection to tox/interpreters.py --- CHANGELOG | 4 +- doc/config.txt | 4 +- tests/test_config.py | 58 ++++++---------- tests/test_interpreters.py | 95 +++++++++++++++++++++++++ tests/test_venv.py | 58 +--------------- tox/_cmdline.py | 2 + tox/_config.py | 44 ++++-------- tox/_venv.py | 73 ++----------------- tox/interpreters.py | 170 +++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 316 insertions(+), 192 deletions(-) create mode 100644 tests/test_interpreters.py create mode 100644 tox/interpreters.py diff --git a/CHANGELOG b/CHANGELOG index 3acaf89f..1e25b09a 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,7 +5,6 @@ installation command with options for dep/pkg install. Thanks Carl Meyer for the PR and docs. - - address issueintroduce python2.5 support by vendoring the virtualenv-1.9.1 script and forcing pip<1.4. Also the default [py25] environment modifies the default installer_command (new config option) to use pip without the "--pre" @@ -36,6 +35,9 @@ - if a HOMEDIR cannot be determined, use the toxinidir. +- refactor interpreter information detection to live in new + tox/interpreters.py file, tests in tests/test_interpreters.py. + 1.5.0 ----------------- diff --git a/doc/config.txt b/doc/config.txt index 449f2a03..5fbeab4d 100644 --- a/doc/config.txt +++ b/doc/config.txt @@ -88,8 +88,8 @@ Complete list of settings that you can put into ``testenv*`` sections: and any defined dependencies. Must contain the substitution key ``{packages}`` which will be replaced by the packages to install. May also contain the substitution key ``{opts}``, which - will be replaced by the ``-i`` option to specify index server - (according to :confval:`indexserver` and the ``:indexserver:dep`` + will be replaced by the ``-i INDEXURL`` option if an index server + is active (see :confval:`indexserver` and the ``:indexserver:dep`` syntax of :confval:`deps`) and the ``--download-cache`` option (if you've specified :confval:`downloadcache`). If your installer does not support ``-i`` and ``--download-cache`` command-line options, diff --git a/tests/test_config.py b/tests/test_config.py index e04a2142..9e35597d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,9 +5,9 @@ import subprocess from textwrap import dedent import py -from tox._config import IniReader, CommandParser -from tox._config import parseconfig -from tox._config import prepare_parse, _split_env +from tox._config import * +from tox._config import _split_env + class TestVenvConfig: def test_config_parsing_minimal(self, tmpdir, newconfig): @@ -473,13 +473,29 @@ class TestConfigTestEnv: assert envconfig.changedir.basename == "abc" assert envconfig.changedir == config.setupdir.join("abc") - def test_install_command_defaults_py25(self, newconfig): + def test_install_command_defaults_py25(self, newconfig, monkeypatch): + from tox.interpreters import Interpreters + def get_info(self, name): + if "x25" in name: + class I: + runnable = True + executable = "python2.5" + version_info = (2,5) + else: + class I: + runnable = False + executable = "python" + return I + monkeypatch.setattr(Interpreters, "get_info", get_info) config = newconfig(""" - [testenv:py25] + [testenv:x25] + basepython = x25 [testenv:py25-x] + basepython = x25 [testenv:py26] + basepython = "python" """) - for name in ("py25", "py25-x"): + for name in ("x25", "py25-x"): env = config.envconfigs[name] assert env.install_command == \ "pip install --insecure {opts} {packages}".split() @@ -714,36 +730,6 @@ class TestConfigTestEnv: assert conf.changedir.basename == 'testing' assert conf.changedir.dirpath().realpath() == tmpdir.realpath() - @pytest.mark.xfailif("sys.platform == 'win32'") - def test_substitution_envsitepackagesdir(self, tmpdir, monkeypatch, - newconfig): - """ - The envsitepackagesdir property is mostly doing system work, - so this test doesn't excercise it very well. - - Usage of envsitepackagesdir on win32/jython will explicitly - throw an exception, - """ - class MockPopen(object): - returncode = 0 - - def __init__(self, *args, **kwargs): - pass - - def communicate(self, *args, **kwargs): - return 'onevalue', 'othervalue' - - monkeypatch.setattr(subprocess, 'Popen', MockPopen) - env = 'py%s' % (''.join(sys.version.split('.')[0:2])) - config = newconfig(""" - [testenv:%s] - commands = {envsitepackagesdir} - """ % (env)) - conf = config.envconfigs[env] - argv = conf.commands - assert argv[0][0] == 'onevalue' - - class TestGlobalOptions: def test_notest(self, newconfig): config = newconfig([], "") diff --git a/tests/test_interpreters.py b/tests/test_interpreters.py new file mode 100644 index 00000000..97cd3f08 --- /dev/null +++ b/tests/test_interpreters.py @@ -0,0 +1,95 @@ +import sys +import os + +import pytest +from tox.interpreters import * + +@pytest.fixture +def interpreters(): + return Interpreters() + +@pytest.mark.skipif("sys.platform != 'win32'") +def test_locate_via_py(monkeypatch): + from tox._venv import locate_via_py + class PseudoPy: + def sysexec(self, *args): + assert args[0] == '-3.2' + assert args[1] == '-c' + # Return value needs to actually exist! + return sys.executable + @staticmethod + def ret_pseudopy(name): + assert name == 'py' + return PseudoPy() + # Monkeypatch py.path.local.sysfind to return PseudoPy + monkeypatch.setattr(py.path.local, 'sysfind', ret_pseudopy) + assert locate_via_py('3', '2') == sys.executable + +def test_find_executable(): + p = find_executable(sys.executable) + assert p == py.path.local(sys.executable) + for ver in [""] + "2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3".split(): + name = "python%s" % ver + if sys.platform == "win32": + pydir = "python%s" % ver.replace(".", "") + x = py.path.local("c:\%s" % pydir) + print (x) + if not x.check(): + continue + else: + if not py.path.local.sysfind(name): + continue + p = find_executable(name) + assert p + popen = py.std.subprocess.Popen([str(p), '-V'], + stderr=py.std.subprocess.PIPE) + stdout, stderr = popen.communicate() + assert ver in py.builtin._totext(stderr, "ascii") + +def test_find_executable_extra(monkeypatch): + @staticmethod + def sysfind(x): + return "hello" + monkeypatch.setattr(py.path.local, "sysfind", sysfind) + t = find_executable("qweqwe") + assert t == "hello" + +def test_run_and_get_interpreter_info(): + name = os.path.basename(sys.executable) + info = run_and_get_interpreter_info(name, sys.executable) + assert info.version_info == tuple(sys.version_info) + assert info.name == name + assert info.executable == sys.executable + +class TestInterpreters: + + def test_get_info_self_exceptions(self, interpreters): + pytest.raises(ValueError, lambda: + interpreters.get_info()) + pytest.raises(ValueError, lambda: + interpreters.get_info(name="12", executable="123")) + + def test_get_executable(self, interpreters): + x = interpreters.get_executable(sys.executable) + assert x == sys.executable + assert not interpreters.get_executable("12l3k1j23") + + def test_get_info__name(self, interpreters): + basename = os.path.basename(sys.executable) + info = interpreters.get_info(basename) + assert info.version_info == tuple(sys.version_info) + assert info.name == basename + assert info.executable == sys.executable + assert info.runnable + + def test_get_info__name_not_exists(self, interpreters): + info = interpreters.get_info("qlwkejqwe") + assert not info.version_info + assert info.name == "qlwkejqwe" + assert not info.executable + assert not info.runnable + + def test_get_sitepackagesdir_error(self, interpreters): + info = interpreters.get_info(sys.executable) + s = interpreters.get_sitepackagesdir(info, "") + assert s diff --git a/tests/test_venv.py b/tests/test_venv.py index 52acc95e..e6792ed5 100644 --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -2,9 +2,7 @@ import py import tox import pytest import os, sys -from tox._venv import VirtualEnv, CreationConfig, getdigest -from tox._venv import find_executable -from tox._venv import _getinterpreterversion +from tox._venv import * py25calls = int(sys.version_info[:2] == (2,5)) @@ -19,52 +17,6 @@ py25calls = int(sys.version_info[:2] == (2,5)) def test_getdigest(tmpdir): assert getdigest(tmpdir) == "0"*32 -@pytest.mark.skipif("sys.platform != 'win32'") -def test_locate_via_py(monkeypatch): - from tox._venv import locate_via_py - class PseudoPy: - def sysexec(self, *args): - assert args[0] == '-3.2' - assert args[1] == '-c' - # Return value needs to actually exist! - return sys.executable - @staticmethod - def ret_pseudopy(name): - assert name == 'py' - return PseudoPy() - # Monkeypatch py.path.local.sysfind to return PseudoPy - monkeypatch.setattr(py.path.local, 'sysfind', ret_pseudopy) - assert locate_via_py('3', '2') == sys.executable - -def test_find_executable(): - p = find_executable(sys.executable) - assert p == py.path.local(sys.executable) - for ver in [""] + "2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3".split(): - name = "python%s" % ver - if sys.platform == "win32": - pydir = "python%s" % ver.replace(".", "") - x = py.path.local("c:\%s" % pydir) - print (x) - if not x.check(): - continue - else: - if not py.path.local.sysfind(name): - continue - p = find_executable(name) - assert p - popen = py.std.subprocess.Popen([str(p), '-V'], - stderr=py.std.subprocess.PIPE) - stdout, stderr = popen.communicate() - assert ver in py.builtin._totext(stderr, "ascii") - -def test_find_executable_extra(monkeypatch): - @staticmethod - def sysfind(x): - return "hello" - monkeypatch.setattr(py.path.local, "sysfind", sysfind) - t = find_executable("qweqwe") - assert t == "hello" - def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): config = newconfig([], """ [testenv:python] @@ -83,10 +35,6 @@ def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): py.test.raises(tox.exception.InterpreterNotFound, venv.getsupportedinterpreter) -def test_getinterpreterversion(): - from distutils.sysconfig import get_python_version - version = _getinterpreterversion(sys.executable) - assert version == get_python_version() def test_create(monkeypatch, mocksession, newconfig): config = newconfig([], """ @@ -107,7 +55,7 @@ def test_create(monkeypatch, mocksession, newconfig): #assert Envconfig.toxworkdir in args assert venv.getcommandpath("easy_install", cwd=py.path.local()) interp = venv._getliveconfig().python - assert interp == venv.getconfigexecutable() + assert interp == venv.envconfig._basepython_info.executable assert venv.path_config.check(exists=False) @pytest.mark.skipif("sys.platform == 'win32'") @@ -243,7 +191,7 @@ def test_install_deps_indexserver(newmocksession): # two different index servers, two calls assert len(l) == 3 args = " ".join(l[0].args) - assert "-i" not in args + assert "-i " not in args assert "dep1" in args args = " ".join(l[1].args) diff --git a/tox/_cmdline.py b/tox/_cmdline.py index d6ee9962..24e47ba5 100644 --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -486,6 +486,8 @@ class Session: for envconfig in self.config.envconfigs.values(): self.report.line("[testenv:%s]" % envconfig.envname, bold=True) self.report.line(" basepython=%s" % envconfig.basepython) + self.report.line(" _basepython_info=%s" % + envconfig._basepython_info) self.report.line(" envpython=%s" % envconfig.envpython) self.report.line(" envtmpdir=%s" % envconfig.envtmpdir) self.report.line(" envbindir=%s" % envconfig.envbindir) diff --git a/tox/_config.py b/tox/_config.py index a12a19e4..a9351d9f 100644 --- a/tox/_config.py +++ b/tox/_config.py @@ -8,6 +8,8 @@ import string import subprocess import textwrap +from tox.interpreters import Interpreters + import py import tox @@ -118,6 +120,7 @@ class Config: def __init__(self): self.envconfigs = {} self.invocationcwd = py.path.local() + self.interpreters = Interpreters() class VenvConfig: def __init__(self, **kw): @@ -141,32 +144,10 @@ class VenvConfig: # no @property to avoid early calling (see callable(subst[key]) checks) def envsitepackagesdir(self): - print_envsitepackagesdir = textwrap.dedent(""" - import sys - from distutils.sysconfig import get_python_lib - sys.stdout.write(get_python_lib(prefix=sys.argv[1])) - """) - - exe = self.getsupportedinterpreter() - # can't use check_output until py27 - proc = subprocess.Popen( - [str(exe), '-c', print_envsitepackagesdir, str(self.envdir)], - stdout=subprocess.PIPE) - odata, edata = proc.communicate() - if proc.returncode: - raise tox.exception.UnsupportedInterpreter( - "Error getting site-packages from %s" % self.basepython) - return odata - - def getconfigexecutable(self): - from tox._venv import find_executable - - python = self.basepython - if not python: - python = sys.executable - x = find_executable(str(python)) - if x: - x = x.realpath() + self.getsupportedinterpreter() # for throwing exceptions + x = self.config.interpreters.get_sitepackagesdir( + info=self._basepython_info, + envdir=self.envdir) return x def getsupportedinterpreter(self): @@ -174,10 +155,11 @@ class VenvConfig: "jython" in self.basepython: raise tox.exception.UnsupportedInterpreter( "Jython/Windows does not support installing scripts") - config_executable = self.getconfigexecutable() - if not config_executable: + info = self.config.interpreters.get_info(self.basepython) + if not info.executable: raise tox.exception.InterpreterNotFound(self.basepython) - return config_executable + return info.executable + testenvprefix = "testenv:" class parseini: @@ -285,6 +267,7 @@ class parseini: else: bp = sys.executable vc.basepython = reader.getdefault(section, "basepython", bp) + vc._basepython_info = config.interpreters.get_info(vc.basepython) reader.addsubstitions(envdir=vc.envdir, envname=vc.envname, envbindir=vc.envbindir, envpython=vc.envpython, envsitepackagesdir=vc.envsitepackagesdir) @@ -336,7 +319,8 @@ class parseini: # need to use --insecure for pip commands because python2.5 # doesn't support SSL pip_default_opts = ["{opts}", "{packages}"] - if "py25" in vc.envname: # XXX too rough check for "python2.5" + info = vc._basepython_info + if info.runnable and info.version_info < (2,6): pip_default_opts.insert(0, "--insecure") else: pip_default_opts.insert(0, "--pre") diff --git a/tox/_venv.py b/tox/_venv.py index 8b451ffe..d2f2ab33 100644 --- a/tox/_venv.py +++ b/tox/_venv.py @@ -145,7 +145,7 @@ class VirtualEnv(object): return "could not install deps %s" %(self.envconfig.deps,) def _getliveconfig(self): - python = self.getconfigexecutable() + python = self.envconfig._basepython_info.executable md5 = getdigest(python) version = tox.__version__ distribute = self.envconfig.distribute @@ -169,9 +169,6 @@ class VirtualEnv(object): l.append(dep) return l - def getconfigexecutable(self): - return self.envconfig.getconfigexecutable() - def getsupportedinterpreter(self): return self.envconfig.getsupportedinterpreter() @@ -180,11 +177,11 @@ class VirtualEnv(object): # return if action is None: action = self.session.newaction(self, "create") + + interpreters = self.envconfig.config.interpreters config_interpreter = self.getsupportedinterpreter() - config_interpreter_version = _getinterpreterversion( - config_interpreter) - use_venv191 = config_interpreter_version < '2.6' - use_pip13 = config_interpreter_version < '2.6' + info = interpreters.get_info(executable=config_interpreter) + use_venv191 = use_pip13 = info.version_info < (2,6) if not use_venv191: f, path, _ = py.std.imp.find_module("virtualenv") f.close() @@ -389,21 +386,6 @@ class VirtualEnv(object): self.session.report.verbosity2("setting PATH=%s" % os.environ["PATH"]) return oldPATH -def _getinterpreterversion(executable): - print_python_version = ( - 'from distutils.sysconfig import get_python_version\n' - 'print(get_python_version())\n') - proc = subprocess.Popen([str(executable), '-c', print_python_version], - stdout=subprocess.PIPE) - odata, edata = proc.communicate() - if proc.returncode: - raise tox.exception.UnsupportedInterpreter( - "Error getting python version from %s" % executable) - if sys.version_info[0] == 3: - string = str - else: - string = lambda x, encoding: str(x) - return string(odata, 'ascii').strip() def getdigest(path): path = py.path.local(path) @@ -411,51 +393,6 @@ def getdigest(path): return "0" * 32 return path.computehash() -if sys.platform != "win32": - def find_executable(name): - return py.path.local.sysfind(name) - -else: - # Exceptions to the usual windows mapping - win32map = { - 'python': sys.executable, - 'jython': "c:\jython2.5.1\jython.bat", - } - def locate_via_py(v_maj, v_min): - ver = "-%s.%s" % (v_maj, v_min) - script = "import sys; print(sys.executable)" - py_exe = py.path.local.sysfind('py') - if py_exe: - try: - exe = py_exe.sysexec(ver, '-c', script).strip() - except py.process.cmdexec.Error: - exe = None - if exe: - exe = py.path.local(exe) - if exe.check(): - return exe - - def find_executable(name): - p = py.path.local.sysfind(name) - if p: - return p - actual = None - # Is this a standard PythonX.Y name? - m = re.match(r"python(\d)\.(\d)", name) - if m: - # The standard names are in predictable places. - actual = r"c:\python%s%s\python.exe" % m.groups() - if not actual: - actual = win32map.get(name, None) - if actual: - actual = py.path.local(actual) - if actual.check(): - return actual - # The standard executables can be found as a last resort via the - # Python launcher py.exe - if m: - locate_via_py(*m.groups()) - def hack_home_env(homedir, index_url=None): # XXX HACK (this could also live with tox itself, consider) diff --git a/tox/interpreters.py b/tox/interpreters.py new file mode 100644 index 00000000..60263a74 --- /dev/null +++ b/tox/interpreters.py @@ -0,0 +1,170 @@ +import sys +import os +import py +import subprocess +import inspect + +class Interpreters: + def __init__(self): + self.name2executable = {} + self.executable2info = {} + + def get_executable(self, name): + """ return path object to the executable for the given + name (e.g. python2.5, python2.7, python etc.) + if name is already an existing path, return name. + If an interpreter cannot be found, return None. + """ + try: + return self.name2executable[name] + except KeyError: + self.name2executable[name] = e = find_executable(name) + return e + + def get_info(self, name=None, executable=None): + if name is None and executable is None: + raise ValueError("need to specify name or executable") + if name: + if executable is not None: + raise ValueError("cannot specify both name, executable") + executable = self.get_executable(name) + if not executable: + return NoInterpreterInfo(name=name) + try: + return self.executable2info[executable] + except KeyError: + info = run_and_get_interpreter_info(name, executable) + self.executable2info[executable] = info + return info + + def get_sitepackagesdir(self, info, envdir): + if not info.executable: + return "" + envdir = str(envdir) + try: + res = exec_on_interpreter(info.executable, + [inspect.getsource(sitepackagesdir), + "print (sitepackagesdir(%r))" % envdir]) + except ExecFailed: + val = sys.exc_info()[1] + print ("execution failed: %s -- %s" %(val.out, val.err)) + return "" + else: + return res["dir"] + +def run_and_get_interpreter_info(name, executable): + assert executable + try: + result = exec_on_interpreter(executable, + [inspect.getsource(pyinfo), "print (pyinfo())"]) + except ExecFailed: + val = sys.exc_info()[1] + return NoInterpreterInfo(name, **val.__dict__) + else: + return InterpreterInfo(name, executable, **result) + +def exec_on_interpreter(executable, source): + if isinstance(source, list): + source = "\n".join(source) + from subprocess import Popen, PIPE + args = [str(executable)] + popen = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) + popen.stdin.write(source.encode("utf8")) + out, err = popen.communicate() + if popen.returncode: + raise ExecFailed(executable, source, out, err) + try: + result = eval(out) + except Exception: + raise ExecFailed(executable, source, out, + "could not decode %r" % out) + return result + +class ExecFailed(Exception): + def __init__(self, executable, source, out, err): + self.executable = executable + self.source = source + self.out = out + self.err = err + +class InterpreterInfo: + runnable = True + + def __init__(self, name, executable, version_info): + assert name and executable and version_info + self.name = name + self.executable = executable + self.version_info = version_info + + def __str__(self): + return "" % ( + self.executable, self.version_info) + +class NoInterpreterInfo: + runnable = False + def __init__(self, name, executable=None, + out=None, err="not found"): + self.name = name + self.executable = executable + self.version_info = None + self.out = out + self.err = err + + def __str__(self): + if self.executable: + return "" + else: + return "" % self.name + +if sys.platform != "win32": + def find_executable(name): + return py.path.local.sysfind(name) + +else: + # Exceptions to the usual windows mapping + win32map = { + 'python': sys.executable, + 'jython': "c:\jython2.5.1\jython.bat", + } + def locate_via_py(v_maj, v_min): + ver = "-%s.%s" % (v_maj, v_min) + script = "import sys; print(sys.executable)" + py_exe = py.path.local.sysfind('py') + if py_exe: + try: + exe = py_exe.sysexec(ver, '-c', script).strip() + except py.process.cmdexec.Error: + exe = None + if exe: + exe = py.path.local(exe) + if exe.check(): + return exe + + def find_executable(name): + p = py.path.local.sysfind(name) + if p: + return p + actual = None + # Is this a standard PythonX.Y name? + m = re.match(r"python(\d)\.(\d)", name) + if m: + # The standard names are in predictable places. + actual = r"c:\python%s%s\python.exe" % m.groups() + if not actual: + actual = win32map.get(name, None) + if actual: + actual = py.path.local(actual) + if actual.check(): + return actual + # The standard executables can be found as a last resort via the + # Python launcher py.exe + if m: + locate_via_py(*m.groups()) + +def pyinfo(): + import sys + return dict(version_info=tuple(sys.version_info)) + +def sitepackagesdir(envdir): + from distutils.sysconfig import get_python_lib + return dict(dir=get_python_lib(envdir)) -- cgit v1.2.1