From d204cac6983b2216bc423c8b67df7a8af3471464 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 13 Nov 2015 10:48:43 +0100 Subject: add various xfailing test cases for env substitution in setenv --- tests/test_config.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 61047c1..44c6e7d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1598,6 +1598,47 @@ class TestHashseedOption: self._check_hashseed(envconfigs["hash2"], '123456789') +class TestSetenv: + @pytest.mark.xfail(reason="fix pending") + def test_setenv_uses_os_environ(self, tmpdir, newconfig, monkeypatch): + monkeypatch.setenv("X", "1") + config = newconfig(""" + [testenv:env1] + setenv = + X = {env:X} + """) + assert config.envconfigs["env1"].setenv["X"] == "1" + + @pytest.mark.xfail(reason="fix pending") + def test_setenv_default_os_environ(self, tmpdir, newconfig, monkeypatch): + monkeypatch.delenv("X", raising=False) + config = newconfig(""" + [testenv:env1] + setenv = + X = {env:X:2} + """) + assert config.envconfigs["env1"].setenv["X"] == "2" + + @pytest.mark.xfail(reason="fix pending") + def test_setenv_uses_other_setenv(self, tmpdir, newconfig): + config = newconfig(""" + [testenv:env1] + setenv = + Y = 5 + X = {env:Y} + """) + assert config.envconfigs["env1"].setenv["X"] == "5" + + @pytest.mark.xfail(reason="fix pending") + def test_setenv_recursive_direct(self, tmpdir, newconfig): + config = newconfig(""" + [testenv:env1] + setenv = + X = {env:X:3} + """) + assert config.envconfigs["env1"].setenv["X"] == "3" + + class TestIndexServer: def test_indexserver(self, tmpdir, newconfig): config = newconfig(""" -- cgit v1.2.1 From 78efe5c87d1ad709d309f998a8c9c252e4fc44a4 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 13 Nov 2015 10:48:45 +0100 Subject: put replacing/substitution into own class --- tox/config.py | 119 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 62 insertions(+), 57 deletions(-) diff --git a/tox/config.py b/tox/config.py index 9f5aae3..6db1e3a 100644 --- a/tox/config.py +++ b/tox/config.py @@ -799,16 +799,6 @@ class IndexServerConfig: is_section_substitution = re.compile("{\[[^{}\s]+\]\S+?}").match -RE_ITEM_REF = re.compile( - r''' - (?[^[:{}]+):)? # optional sub_type for special rules - (?P[^{}]*) # substitution key - [}] - ''', - re.VERBOSE) - - class SectionReader: def __init__(self, section_name, cfgparser, fallbacksections=None, factors=()): self.section_name = section_name @@ -888,11 +878,7 @@ class SectionReader: x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - self._subststack.append((self.section_name, name)) - try: - x = self._replace(x) - finally: - assert self._subststack.pop() == (self.section_name, name) + x = self._replace(x, name=name) # print "getstring", self.section_name, name, "returned", repr(x) return x @@ -909,8 +895,61 @@ class SectionReader: lines = s.strip().splitlines() return '\n'.join(filter(None, map(factor_line, lines))) + def _replace(self, value, name=None, section_name=None): + if '{' not in value: + return value + + section_name = section_name if section_name else self.section_name + self._subststack.append((section_name, name)) + try: + return Replacer(self).do_replace(value) + finally: + assert self._subststack.pop() == (section_name, name) + + +class Replacer: + RE_ITEM_REF = re.compile( + r''' + (?[^[:{}]+):)? # optional sub_type for special rules + (?P[^{}]*) # substitution key + [}] + ''', + re.VERBOSE) + + + def __init__(self, reader, opt_replace_env=True): + self.reader = reader + self.opt_replace_env = opt_replace_env + + def do_replace(self, x): + return self.RE_ITEM_REF.sub(self._replace_match, x) + + def _replace_match(self, match): + g = match.groupdict() + + # special case: opts and packages. Leave {opts} and + # {packages} intact, they are replaced manually in + # _venv.VirtualEnv.run_install_command. + sub_value = g['substitution_value'] + if sub_value in ('opts', 'packages'): + return '{%s}' % sub_value + + try: + sub_type = g['sub_type'] + except KeyError: + raise tox.exception.ConfigError( + "Malformed substitution; no substitution type provided") + + if sub_type == "env": + assert self.opt_replace_env + return self._replace_env(match) + if sub_type != None: + raise tox.exception.ConfigError("No support for the %s substitution type" % sub_type) + return self._replace_substitution(match) + def _replace_env(self, match): - env_list = self.getdict('setenv') + env_list = self.reader.getdict('setenv') match_value = match.group('substitution_value') if not match_value: raise tox.exception.ConfigError( @@ -938,60 +977,26 @@ class SectionReader: if key.startswith("[") and "]" in key: i = key.find("]") section, item = key[1:i], key[i + 1:] - if section in self._cfg and item in self._cfg[section]: - if (section, item) in self._subststack: + cfg = self.reader._cfg + if section in cfg and item in cfg[section]: + if (section, item) in self.reader._subststack: raise ValueError('%s already in %s' % ( - (section, item), self._subststack)) - x = str(self._cfg[section][item]) - self._subststack.append((section, item)) - try: - return self._replace(x) - finally: - self._subststack.pop() + (section, item), self.reader._subststack)) + x = str(cfg[section][item]) + return self.reader._replace(x, name=item, section_name=section) raise tox.exception.ConfigError( "substitution key %r not found" % key) def _replace_substitution(self, match): sub_key = match.group('substitution_value') - val = self._subs.get(sub_key, None) + val = self.reader._subs.get(sub_key, None) if val is None: val = self._substitute_from_other_section(sub_key) if py.builtin.callable(val): val = val() return str(val) - def _replace_match(self, match): - g = match.groupdict() - - # special case: opts and packages. Leave {opts} and - # {packages} intact, they are replaced manually in - # _venv.VirtualEnv.run_install_command. - sub_value = g['substitution_value'] - if sub_value in ('opts', 'packages'): - return '{%s}' % sub_value - - handlers = { - 'env': self._replace_env, - None: self._replace_substitution, - } - try: - sub_type = g['sub_type'] - except KeyError: - raise tox.exception.ConfigError( - "Malformed substitution; no substitution type provided") - - try: - handler = handlers[sub_type] - except KeyError: - raise tox.exception.ConfigError("No support for the %s substitution type" % sub_type) - - return handler(match) - - def _replace(self, x): - if '{' in x: - return RE_ITEM_REF.sub(self._replace_match, x) - return x class _ArgvlistReader: -- cgit v1.2.1 From 483d1a4b61b7088b7ba90894683ace252c4f97b7 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 13 Nov 2015 10:49:05 +0100 Subject: address issue285 but not fully resolve it: setenv processing now works, making all config tests pass except for one substitution issue which requires yet some more logic, probably. --- tests/test_config.py | 32 +++++++-------- tox/config.py | 109 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 44c6e7d..9ba2baa 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -283,18 +283,9 @@ class TestIniParserAgainstCommandsKey: commands = ls {env:TEST} """) - reader = SectionReader("testenv:py27", config._cfg) - x = reader.getargvlist("commands") - assert x == [ - "ls testvalue".split() - ] - assert x != [ - "ls {env:TEST}".split() - ] - y = reader.getargvlist("setenv") - assert y == [ - "TEST=testvalue".split() - ] + envconfig = config.envconfigs["py27"] + assert envconfig.commands == [["ls", "testvalue"]] + assert envconfig.setenv["TEST"] == "testvalue" class TestIniParser: @@ -1599,7 +1590,19 @@ class TestHashseedOption: class TestSetenv: - @pytest.mark.xfail(reason="fix pending") + def test_getdict_lazy(self, tmpdir, newconfig): + config = newconfig(""" + [testenv:X] + key0 = + key1 = {env:X} + key2 = {env:X:1} + """) + envconfig = config.envconfigs["X"] + val = envconfig._reader.getdict_lazy("key0") + assert val == {"key1": "{env:X}", + "key2": "{env:X:1}"} + + def test_setenv_uses_os_environ(self, tmpdir, newconfig, monkeypatch): monkeypatch.setenv("X", "1") config = newconfig(""" @@ -1609,7 +1612,6 @@ class TestSetenv: """) assert config.envconfigs["env1"].setenv["X"] == "1" - @pytest.mark.xfail(reason="fix pending") def test_setenv_default_os_environ(self, tmpdir, newconfig, monkeypatch): monkeypatch.delenv("X", raising=False) config = newconfig(""" @@ -1619,7 +1621,6 @@ class TestSetenv: """) assert config.envconfigs["env1"].setenv["X"] == "2" - @pytest.mark.xfail(reason="fix pending") def test_setenv_uses_other_setenv(self, tmpdir, newconfig): config = newconfig(""" [testenv:env1] @@ -1629,7 +1630,6 @@ class TestSetenv: """) assert config.envconfigs["env1"].setenv["X"] == "5" - @pytest.mark.xfail(reason="fix pending") def test_setenv_recursive_direct(self, tmpdir, newconfig): config = newconfig(""" [testenv:env1] diff --git a/tox/config.py b/tox/config.py index 6db1e3a..7a500f0 100644 --- a/tox/config.py +++ b/tox/config.py @@ -323,11 +323,39 @@ def tox_addoption(parser): parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") - # add various core venv interpreter attributes parser.add_testenv_attribute( name="envdir", type="path", default="{toxworkdir}/{envname}", help="venv directory") + # add various core venv interpreter attributes + def setenv(testenv_config, value): + setenv = value + reader = testenv_config._reader + + # we need to resolve environment variable substitution + + replacing = [] # for detecting direct recursion + def setenv_reader(name): + if name in setenv and name not in replacing: + return setenv[name] + return os.environ.get(name) + reader.set_envreader(setenv_reader) + + for name, value in setenv.items(): + replacing.append(name) + setenv[name] = reader._replace(value) + replacing.pop() + + config = testenv_config.config + if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: + setenv['PYTHONHASHSEED'] = config.hashseed + return setenv + + parser.add_testenv_attribute( + name="setenv", type="dict_lazy", postprocess=setenv, + help="list of X=Y lines with environment variable settings") + + def basepython_default(testenv_config, value): if value is None: for f in testenv_config.factors: @@ -385,17 +413,6 @@ def tox_addoption(parser): name="recreate", type="bool", default=False, postprocess=recreate, help="always recreate this test environment.") - def setenv(testenv_config, value): - setenv = value - config = testenv_config.config - if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: - setenv['PYTHONHASHSEED'] = config.hashseed - return setenv - - parser.add_testenv_attribute( - name="setenv", type="dict", postprocess=setenv, - help="list of X=Y lines with environment variable settings") - def passenv(testenv_config, value): # Flatten the list to deal with space-separated values. value = list( @@ -518,21 +535,15 @@ class TestenvConfig: @property def envbindir(self): """ path to directory where scripts/binaries reside. """ - if (sys.platform == "win32" - and "jython" not in self.basepython - and "pypy" not in self.basepython): + if sys.platform == "win32": return self.envdir.join("Scripts") else: return self.envdir.join("bin") @property def envpython(self): - """ path to python/jython executable. """ - if "jython" in str(self.basepython): - name = "jython" - else: - name = "python" - return self.envbindir.join(name) + """ path to python executable. """ + return self.envbindir.join(self.basepython) # no @property to avoid early calling (see callable(subst[key]) checks) def envsitepackagesdir(self): @@ -699,7 +710,7 @@ class parseini: for env_attr in config._testenv_attr: atype = env_attr.type - if atype in ("bool", "path", "string", "dict", "argv", "argvlist"): + if atype in ("bool", "path", "string", "dict", "dict_lazy", "argv", "argvlist"): meth = getattr(reader, "get" + atype) res = meth(env_attr.name, env_attr.default) elif atype == "space-separated-list": @@ -807,6 +818,13 @@ class SectionReader: self.factors = factors self._subs = {} self._subststack = [] + self._envreader = os.environ.get + + def set_envreader(self, envreader): + self._envreader = envreader + + def get_environ_value(self, name): + return self._envreader(name) def addsubstitutions(self, _posargs=None, **kw): self._subs.update(kw) @@ -826,17 +844,24 @@ class SectionReader: return [x.strip() for x in s.split(sep) if x.strip()] def getdict(self, name, default=None, sep="\n"): - s = self.getstring(name, None) - if s is None: + value = self.getstring(name, None) + return self._getdict(value, default=default, sep=sep) + + def getdict_lazy(self, name, default=None, sep="\n"): + value = self.getstring(name, None, replace="noenv") + return self._getdict(value, default=default, sep=sep) + + def _getdict(self, value, default, sep): + if value is None: return default or {} - value = {} - for line in s.split(sep): + d = {} + for line in value.split(sep): if line.strip(): name, rest = line.split('=', 1) - value[name.strip()] = rest.strip() + d[name.strip()] = rest.strip() - return value + return d def getbool(self, name, default=None): s = self.getstring(name, default) @@ -878,7 +903,7 @@ class SectionReader: x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - x = self._replace(x, name=name) + x = self._replace(x, name=name, opt_replace_env=(replace!="noenv")) # print "getstring", self.section_name, name, "returned", repr(x) return x @@ -895,14 +920,14 @@ class SectionReader: lines = s.strip().splitlines() return '\n'.join(filter(None, map(factor_line, lines))) - def _replace(self, value, name=None, section_name=None): + def _replace(self, value, name=None, section_name=None, opt_replace_env=True): if '{' not in value: return value section_name = section_name if section_name else self.section_name self._subststack.append((section_name, name)) try: - return Replacer(self).do_replace(value) + return Replacer(self, opt_replace_env=opt_replace_env).do_replace(value) finally: assert self._subststack.pop() == (section_name, name) @@ -918,7 +943,7 @@ class Replacer: re.VERBOSE) - def __init__(self, reader, opt_replace_env=True): + def __init__(self, reader, opt_replace_env): self.reader = reader self.opt_replace_env = opt_replace_env @@ -942,14 +967,14 @@ class Replacer: "Malformed substitution; no substitution type provided") if sub_type == "env": - assert self.opt_replace_env - return self._replace_env(match) + if self.opt_replace_env: + return self._replace_env(match) + return "{env:%s}" %(g["substitution_value"]) if sub_type != None: raise tox.exception.ConfigError("No support for the %s substitution type" % sub_type) return self._replace_substitution(match) def _replace_env(self, match): - env_list = self.reader.getdict('setenv') match_value = match.group('substitution_value') if not match_value: raise tox.exception.ConfigError( @@ -963,15 +988,14 @@ class Replacer: else: envkey = match_value - if envkey not in os.environ and default is None: - if envkey not in env_list and default is None: + envvalue = self.reader.get_environ_value(envkey) + if envvalue is None: + if default is None: raise tox.exception.ConfigError( "substitution env:%r: unknown environment variable %r" % (envkey, envkey)) - if envkey in os.environ: - return os.environ.get(envkey, default) - else: - return env_list.get(envkey, default) + return default + return envvalue def _substitute_from_other_section(self, key): if key.startswith("[") and "]" in key: @@ -983,7 +1007,8 @@ class Replacer: raise ValueError('%s already in %s' % ( (section, item), self.reader._subststack)) x = str(cfg[section][item]) - return self.reader._replace(x, name=item, section_name=section) + return self.reader._replace(x, name=item, section_name=section, + opt_replace_env=self.opt_replace_env) raise tox.exception.ConfigError( "substitution key %r not found" % key) -- cgit v1.2.1 From 29861cd9565db7f47d6e40e1245c12c2744e6bcf Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 13 Nov 2015 10:50:44 +0100 Subject: add changelog --- CHANGELOG | 5 +++++ setup.py | 2 +- tox/__init__.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 11ecb1f..6e95ce8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +2.2.2.dev +--------- + +- fix issue285 (WIP) setenv processing with self-references + 2.2.1 ----- diff --git a/setup.py b/setup.py index a398ade..3bff9fa 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def main(): description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='2.2.1', + version='2.2.2.dev1', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff --git a/tox/__init__.py b/tox/__init__.py index c1e74fb..f69bc2d 100644 --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '2.2.1' +__version__ = '2.2.2.dev1' from .hookspecs import hookspec, hookimpl # noqa -- cgit v1.2.1 From e90ef03d7cca2fe120b9a7169e8f2aee1f9300a4 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 7 Dec 2015 12:38:55 +0100 Subject: refactor setenv processing into its own class so that we can cleanly implement lazyness and get rid of all kinds of ordering problems. --- CHANGELOG | 5 +- tests/test_config.py | 30 +++++++++--- tox.ini | 2 +- tox/config.py | 134 ++++++++++++++++++++++++++++++++------------------- 4 files changed, 111 insertions(+), 60 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index f3b30ec..1a395fc 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,10 @@ 2.3.0 (unreleased) ----- -- fix issue285 (WIP) setenv processing with self-references +- fix issue285 make setenv processing fully lazy to fix regressions + of tox-2.2.X and so that we can now have testenv attributes like + "basepython" depend on environment variables that are set in + a setenv section. - allow "#" in commands. This is slightly incompatible with commands sections that used a comment after a "\" line continuation. diff --git a/tests/test_config.py b/tests/test_config.py index 462e33c..573fdfe 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -644,7 +644,7 @@ class TestConfigTestEnv: assert envconfig.usedevelop is False assert envconfig.ignore_errors is False assert envconfig.envlogdir == envconfig.envdir.join("log") - assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] + assert list(envconfig.setenv.definitions.keys()) == ['PYTHONHASHSEED'] hashseed = envconfig.setenv['PYTHONHASHSEED'] assert isinstance(hashseed, str) # The following line checks that hashseed parses to an integer. @@ -1516,7 +1516,7 @@ class TestHashseedOption: return envconfigs["python"] def _check_hashseed(self, envconfig, expected): - assert envconfig.setenv == {'PYTHONHASHSEED': expected} + assert envconfig.setenv['PYTHONHASHSEED'] == expected def _check_testenv(self, newconfig, expected, args=None, tox_ini=None): envconfig = self._get_envconfig(newconfig, args=args, tox_ini=tox_ini) @@ -1565,7 +1565,7 @@ class TestHashseedOption: def test_noset(self, tmpdir, newconfig): args = ['--hashseed', 'noset'] envconfig = self._get_envconfig(newconfig, args=args) - assert envconfig.setenv == {} + assert not envconfig.setenv.definitions def test_noset_with_setenv(self, tmpdir, newconfig): tox_ini = """ @@ -1610,18 +1610,32 @@ class TestHashseedOption: class TestSetenv: - def test_getdict_lazy(self, tmpdir, newconfig): + def test_getdict_lazy(self, tmpdir, newconfig, monkeypatch): + monkeypatch.setenv("X", "2") config = newconfig(""" [testenv:X] key0 = key1 = {env:X} - key2 = {env:X:1} + key2 = {env:Y:1} """) envconfig = config.envconfigs["X"] - val = envconfig._reader.getdict_lazy("key0") - assert val == {"key1": "{env:X}", - "key2": "{env:X:1}"} + val = envconfig._reader.getdict_setenv("key0") + assert val["key1"] == "2" + assert val["key2"] == "1" + def test_getdict_lazy_update(self, tmpdir, newconfig, monkeypatch): + monkeypatch.setenv("X", "2") + config = newconfig(""" + [testenv:X] + key0 = + key1 = {env:X} + key2 = {env:Y:1} + """) + envconfig = config.envconfigs["X"] + val = envconfig._reader.getdict_setenv("key0") + d = {} + d.update(val) + assert d == {"key1": "2", "key2": "1"} def test_setenv_uses_os_environ(self, tmpdir, newconfig, monkeypatch): monkeypatch.setenv("X", "1") diff --git a/tox.ini b/tox.ini index d78bcbd..e79b882 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ envlist=py27,py26,py34,py33,pypy,flakes,py26-bare commands=echo {posargs} [testenv] -commands= py.test --timeout=180 {posargs} +commands= py.test --timeout=180 {posargs:tests} deps=pytest>=2.3.5 pytest-timeout diff --git a/tox/config.py b/tox/config.py index 480704b..4fa0a0a 100644 --- a/tox/config.py +++ b/tox/config.py @@ -26,6 +26,8 @@ for version in '26,27,32,33,34,35,36'.split(','): hookimpl = pluggy.HookimplMarker("tox") +_dummy = object() + def get_plugin_manager(): # initialize plugin manager @@ -253,6 +255,47 @@ class CountAction(argparse.Action): setattr(namespace, self.dest, 0) +class SetenvDict: + def __init__(self, dict, reader): + self.reader = reader + self.definitions = dict + self.resolved = {} + self._lookupstack = [] + + def __contains__(self, name): + return name in self.definitions + + def get(self, name, default=None): + try: + return self.resolved[name] + except KeyError: + try: + if name in self._lookupstack: + raise KeyError("recursion") + val = self.definitions[name] + except KeyError: + return os.environ.get(name, default) + self._lookupstack.append(name) + try: + self.resolved[name] = res = self.reader._replace(val) + finally: + self._lookupstack.pop() + return res + + def __getitem__(self, name): + x = self.get(name, _dummy) + if x is _dummy: + raise KeyError(name) + return x + + def keys(self): + return self.definitions.keys() + + def __setitem__(self, name, value): + self.definitions[name] = value + self.resolved[name] = value + + @hookimpl def tox_addoption(parser): # formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -330,32 +373,15 @@ def tox_addoption(parser): # add various core venv interpreter attributes def setenv(testenv_config, value): setenv = value - reader = testenv_config._reader - - # we need to resolve environment variable substitution - - replacing = [] # for detecting direct recursion - def setenv_reader(name): - if name in setenv and name not in replacing: - return setenv[name] - return os.environ.get(name) - reader.set_envreader(setenv_reader) - - for name, value in setenv.items(): - replacing.append(name) - setenv[name] = reader._replace(value) - replacing.pop() - config = testenv_config.config if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: setenv['PYTHONHASHSEED'] = config.hashseed return setenv parser.add_testenv_attribute( - name="setenv", type="dict_lazy", postprocess=setenv, + name="setenv", type="dict_setenv", postprocess=setenv, help="list of X=Y lines with environment variable settings") - def basepython_default(testenv_config, value): if value is None: for f in testenv_config.factors: @@ -532,21 +558,33 @@ class TestenvConfig: self.factors = factors self._reader = reader - @property - def envbindir(self): + def get_envbindir(self): """ path to directory where scripts/binaries reside. """ - if sys.platform == "win32": + if (sys.platform == "win32" + and "jython" not in self.basepython + and "pypy" not in self.basepython): return self.envdir.join("Scripts") else: return self.envdir.join("bin") + @property + def envbindir(self): + return self.get_envbindir() + @property def envpython(self): """ path to python executable. """ - return self.envbindir.join(self.basepython) + return self.get_envpython() - # no @property to avoid early calling (see callable(subst[key]) checks) - def envsitepackagesdir(self): + def get_envpython(self): + """ path to python/jython executable. """ + if "jython" in str(self.basepython): + name = "jython" + else: + name = "python" + return self.envbindir.join(name) + + def get_envsitepackagesdir(self): """ return sitepackagesdir of the virtualenv environment. (only available during execution, not parsing) """ @@ -707,10 +745,13 @@ class parseini: vc = TestenvConfig(config=config, envname=name, factors=factors, reader=reader) reader.addsubstitutions(**subs) reader.addsubstitutions(envname=name) + reader.addsubstitutions(envbindir=vc.get_envbindir, + envsitepackagesdir=vc.get_envsitepackagesdir, + envpython=vc.get_envpython) for env_attr in config._testenv_attr: atype = env_attr.type - if atype in ("bool", "path", "string", "dict", "dict_lazy", "argv", "argvlist"): + if atype in ("bool", "path", "string", "dict", "dict_setenv", "argv", "argvlist"): meth = getattr(reader, "get" + atype) res = meth(env_attr.name, env_attr.default) elif atype == "space-separated-list": @@ -727,9 +768,6 @@ class parseini: if atype == "path": reader.addsubstitutions(**{env_attr.name: res}) - if env_attr.name == "basepython": - reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, - envsitepackagesdir=vc.envsitepackagesdir) return vc def _getenvdata(self, reader): @@ -818,13 +856,12 @@ class SectionReader: self.factors = factors self._subs = {} self._subststack = [] - self._envreader = os.environ.get - - def set_envreader(self, envreader): - self._envreader = envreader + self._setenv = None def get_environ_value(self, name): - return self._envreader(name) + if self._setenv is None: + return os.environ.get(name) + return self._setenv.get(name) def addsubstitutions(self, _posargs=None, **kw): self._subs.update(kw) @@ -847,9 +884,11 @@ class SectionReader: value = self.getstring(name, None) return self._getdict(value, default=default, sep=sep) - def getdict_lazy(self, name, default=None, sep="\n"): - value = self.getstring(name, None, replace="noenv") - return self._getdict(value, default=default, sep=sep) + def getdict_setenv(self, name, default=None, sep="\n"): + value = self.getstring(name, None, replace=False) + definitions = self._getdict(value, default=default, sep=sep) + self._setenv = SetenvDict(definitions, reader=self) + return self._setenv def _getdict(self, value, default, sep): if value is None: @@ -903,7 +942,7 @@ class SectionReader: x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - x = self._replace(x, name=name, opt_replace_env=(replace!="noenv")) + x = self._replace(x, name=name) # print "getstring", self.section_name, name, "returned", repr(x) return x @@ -920,14 +959,14 @@ class SectionReader: lines = s.strip().splitlines() return '\n'.join(filter(None, map(factor_line, lines))) - def _replace(self, value, name=None, section_name=None, opt_replace_env=True): + def _replace(self, value, name=None, section_name=None): if '{' not in value: return value section_name = section_name if section_name else self.section_name self._subststack.append((section_name, name)) try: - return Replacer(self, opt_replace_env=opt_replace_env).do_replace(value) + return Replacer(self).do_replace(value) finally: assert self._subststack.pop() == (section_name, name) @@ -942,10 +981,8 @@ class Replacer: ''', re.VERBOSE) - - def __init__(self, reader, opt_replace_env): + def __init__(self, reader): self.reader = reader - self.opt_replace_env = opt_replace_env def do_replace(self, x): return self.RE_ITEM_REF.sub(self._replace_match, x) @@ -967,11 +1004,10 @@ class Replacer: "Malformed substitution; no substitution type provided") if sub_type == "env": - if self.opt_replace_env: - return self._replace_env(match) - return "{env:%s}" %(g["substitution_value"]) - if sub_type != None: - raise tox.exception.ConfigError("No support for the %s substitution type" % sub_type) + return self._replace_env(match) + if sub_type is not None: + raise tox.exception.ConfigError( + "No support for the %s substitution type" % sub_type) return self._replace_substitution(match) def _replace_env(self, match): @@ -1007,8 +1043,7 @@ class Replacer: raise ValueError('%s already in %s' % ( (section, item), self.reader._subststack)) x = str(cfg[section][item]) - return self.reader._replace(x, name=item, section_name=section, - opt_replace_env=self.opt_replace_env) + return self.reader._replace(x, name=item, section_name=section) raise tox.exception.ConfigError( "substitution key %r not found" % key) @@ -1023,7 +1058,6 @@ class Replacer: return str(val) - class _ArgvlistReader: @classmethod def getargvlist(cls, reader, value): -- cgit v1.2.1 From 1443960a5e6f016aa96e77a13322c5e904b31ed7 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 7 Dec 2015 12:39:34 +0100 Subject: reshuffle tests related to setenv processing and integrate nelfin's cross-section test but mark it as xfailing because i am not sure we need to go to the trouble --- CHANGELOG | 5 +-- setup.py | 2 +- tests/test_config.py | 95 ++++++++++++++++++++++++++++++---------------------- tox/__init__.py | 2 +- tox/config.py | 5 +-- 5 files changed, 63 insertions(+), 46 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 1a395fc..d22738a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,11 @@ 2.3.0 (unreleased) ----- -- fix issue285 make setenv processing fully lazy to fix regressions +- fix issue285: make setenv processing fully lazy to fix regressions of tox-2.2.X and so that we can now have testenv attributes like "basepython" depend on environment variables that are set in - a setenv section. + a setenv section. Thanks Nelfin for some tests and initial + work on a PR. - allow "#" in commands. This is slightly incompatible with commands sections that used a comment after a "\" line continuation. diff --git a/setup.py b/setup.py index 3a0ab18..350a098 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def main(): description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='2.3.0.dev1', + version='2.3.0.dev2', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff --git a/tests/test_config.py b/tests/test_config.py index 573fdfe..f727a1f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -735,46 +735,6 @@ class TestConfigTestEnv: if bp == "jython": assert envconfig.envpython == envconfig.envbindir.join(bp) - def test_setenv_overrides(self, tmpdir, newconfig): - config = newconfig(""" - [testenv] - setenv = - PYTHONPATH = something - ANOTHER_VAL=else - """) - assert len(config.envconfigs) == 1 - envconfig = config.envconfigs['python'] - assert 'PYTHONPATH' in envconfig.setenv - assert 'ANOTHER_VAL' in envconfig.setenv - assert envconfig.setenv['PYTHONPATH'] == 'something' - assert envconfig.setenv['ANOTHER_VAL'] == 'else' - - def test_setenv_with_envdir_and_basepython(self, tmpdir, newconfig): - config = newconfig(""" - [testenv] - setenv = - VAL = {envdir} - basepython = {env:VAL} - """) - assert len(config.envconfigs) == 1 - envconfig = config.envconfigs['python'] - assert 'VAL' in envconfig.setenv - assert envconfig.setenv['VAL'] == envconfig.envdir - assert envconfig.basepython == envconfig.envdir - - def test_setenv_ordering_1(self, tmpdir, newconfig): - config = newconfig(""" - [testenv] - setenv= - VAL={envdir} - commands=echo {env:VAL} - """) - assert len(config.envconfigs) == 1 - envconfig = config.envconfigs['python'] - assert 'VAL' in envconfig.setenv - assert envconfig.setenv['VAL'] == envconfig.envdir - assert str(envconfig.envdir) in envconfig.commands[0] - @pytest.mark.parametrize("plat", ["win32", "linux2"]) def test_passenv_as_multiline_list(self, tmpdir, newconfig, monkeypatch, plat): monkeypatch.setattr(sys, "platform", plat) @@ -1672,6 +1632,61 @@ class TestSetenv: """) assert config.envconfigs["env1"].setenv["X"] == "3" + def test_setenv_overrides(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + setenv = + PYTHONPATH = something + ANOTHER_VAL=else + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert 'PYTHONPATH' in envconfig.setenv + assert 'ANOTHER_VAL' in envconfig.setenv + assert envconfig.setenv['PYTHONPATH'] == 'something' + assert envconfig.setenv['ANOTHER_VAL'] == 'else' + + def test_setenv_with_envdir_and_basepython(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + setenv = + VAL = {envdir} + basepython = {env:VAL} + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert 'VAL' in envconfig.setenv + assert envconfig.setenv['VAL'] == envconfig.envdir + assert envconfig.basepython == envconfig.envdir + + def test_setenv_ordering_1(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + setenv= + VAL={envdir} + commands=echo {env:VAL} + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert 'VAL' in envconfig.setenv + assert envconfig.setenv['VAL'] == envconfig.envdir + assert str(envconfig.envdir) in envconfig.commands[0] + + @pytest.mark.xfail(reason="we don't implement cross-section substitution for setenv") + def test_setenv_cross_section_subst(self, monkeypatch, newconfig): + """test that we can do cross-section substitution with setenv""" + monkeypatch.delenv('TEST', raising=False) + config = newconfig(""" + [section] + x = + NOT_TEST={env:TEST:defaultvalue} + + [testenv] + setenv = {[section]x} + """) + envconfig = config.envconfigs["python"] + assert envconfig.setenv["NOT_TEST"] == "defaultvalue" + class TestIndexServer: def test_indexserver(self, tmpdir, newconfig): diff --git a/tox/__init__.py b/tox/__init__.py index 837768c..443ce8f 100644 --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '2.3.0.dev1' +__version__ = '2.3.0.dev2' from .hookspecs import hookspec, hookimpl # noqa diff --git a/tox/config.py b/tox/config.py index 4fa0a0a..d34a597 100644 --- a/tox/config.py +++ b/tox/config.py @@ -271,7 +271,7 @@ class SetenvDict: except KeyError: try: if name in self._lookupstack: - raise KeyError("recursion") + raise KeyError(name) val = self.definitions[name] except KeyError: return os.environ.get(name, default) @@ -1028,7 +1028,8 @@ class Replacer: if envvalue is None: if default is None: raise tox.exception.ConfigError( - "substitution env:%r: unknown environment variable %r" % + "substitution env:%r: unknown environment variable %r " + " or recursive definition." % (envkey, envkey)) return default return envvalue -- cgit v1.2.1