diff options
| -rw-r--r-- | src/engine/SCons/EnvironmentValue.py | 28 | ||||
| -rw-r--r-- | src/engine/SCons/EnvironmentValueTests.py | 9 | ||||
| -rw-r--r-- | src/engine/SCons/EnvironmentValues.py | 73 | ||||
| -rw-r--r-- | src/engine/SCons/EnvironmentValuesSubstTests.py | 9 |
4 files changed, 79 insertions, 40 deletions
diff --git a/src/engine/SCons/EnvironmentValue.py b/src/engine/SCons/EnvironmentValue.py index 7626e5567..4f283a8c5 100644 --- a/src/engine/SCons/EnvironmentValue.py +++ b/src/engine/SCons/EnvironmentValue.py @@ -3,9 +3,11 @@ import re from collections import UserDict, UserList from numbers import Number -from SCons.Util import is_String, is_Sequence, CLVar +from SCons.Util import is_String, is_Sequence, CLVar, flatten from SCons.Subst import AllowableExceptions, raise_exception +# remove for non py3 +from enum import Enum _debug = False if _debug: @@ -63,7 +65,7 @@ class SubstModes(object): SUBST_LIST = 3 -class ValueTypes(object): +class ValueTypes(Enum): """ Enum to store what type of value the variable holds. """ @@ -127,6 +129,7 @@ class EnvironmentValue(object): We're going to keep track of variables which feed into this values evaluation """ + __slots__ = ['_parsed', 'all_dependencies', 'value', 'var_type', 'depends_on', 'value', 'cached'] all_values = {} @classmethod @@ -145,7 +148,8 @@ class EnvironmentValue(object): return no def __init__(self, value): - # TODO: Should cache initialziation by keeping hash all previous values, since we don't evaluate in context in the Value itself. + # TODO: Should cache initialization by keeping hash all previous values, since we don't evaluate + # in context in the Value itself. self.value = value self.var_type = ValueTypes.UNKNOWN @@ -225,12 +229,14 @@ class EnvironmentValue(object): # like dict, tuple, list self.var_type = ValueTypes.COLLECTION - if len(self.value) == 1 and isinstance(self.value, (list, UserDict, tuple))\ - and '$' not in self.value[0]: - # Short cut if we only have 1 value and it has no $'s - self.cached = (str(self.value[0]), str(self.value[0])) - + # This fails if value is list of list [['a','b','c']] + # if len(self.value) == 1 and isinstance(self.value, (list, UserDict, tuple))\ + # and '$' not in self.value[0]: + # # Short cut if we only have 1 value and it has no $'s + # self.cached = (str(self.value[0]), str(self.value[0])) + self._parsed = flatten(self.value) + self.find_dependencies() #TODO Handle lists #TODO Handle Dicts elif isinstance(self.value, Number): @@ -285,9 +291,9 @@ class EnvironmentValue(object): # TARGET[0] -> TARGET, TARGET.abspath -> TARGET if '.' in p: - p = p.split('.',1)[0] + p = p.split('.', 1)[0] if '[' in p: - p = p.split('[',1)[0] + p = p.split('[', 1)[0] self.depends_on.add(p) @@ -349,7 +355,7 @@ class EnvironmentValue(object): self.debug_print_parsed_parts(all_dependencies) - depend_list = [v for (t,v,i) in all_dependencies + depend_list = [v for (t, v, i) in all_dependencies if t in (ValueTypes.VARIABLE_OR_CALLABLE, ValueTypes.VARIABLE, ValueTypes.CALLABLE)] self.depends_on = self.depends_on.union(set(depend_list)) diff --git a/src/engine/SCons/EnvironmentValueTests.py b/src/engine/SCons/EnvironmentValueTests.py index 44a596345..2417ddccf 100644 --- a/src/engine/SCons/EnvironmentValueTests.py +++ b/src/engine/SCons/EnvironmentValueTests.py @@ -98,6 +98,15 @@ class TestEnvironmentValue(unittest.TestCase): self.assertEqual(one.var_type, ValueTypes.COLLECTION) self.assertEqual(one.depends_on, set()) + def test_list_in_list_value(self): + # TODO Make this work correctly. + value = [['gcc', 'g++', 'applelink', 'ar', 'libtool', 'as', 'xcode']] + one = EnvironmentValue(value) + self.assertEqual(one._parsed, []) + self.assertEqual(one.value, value) + self.assertEqual(one.var_type, ValueTypes.COLLECTION) + self.assertEqual(one.depends_on, set()) + def test_tuple_value(self): value = (('distmod', '$MONGO_DISTMOD', True, True), ('distarch', '$MONGO_DISTARCH', True, True), ('cc', '$CC_VERSION', True, False), ('ccflags', '$CCFLAGS', True, False), diff --git a/src/engine/SCons/EnvironmentValues.py b/src/engine/SCons/EnvironmentValues.py index dec9c3bcc..81f535663 100644 --- a/src/engine/SCons/EnvironmentValues.py +++ b/src/engine/SCons/EnvironmentValues.py @@ -22,7 +22,7 @@ AllowableExceptions = (KeyError,) # space characters in the string result from the scons_subst() function. space_sep = re.compile(r'[\t ]+(?![^{]*})') -_debug = False +_debug = True if _debug: def debug(fmt, *args): # format when needed @@ -136,6 +136,7 @@ class EnvironmentValues(object): """ A class to hold all the environment variables """ + __slots__ = ['values', '_dict', 'key_set', 'cached_values'] def __init__(self, **kw): self.values = dict((k, EnvironmentValue(v)) for k, v in kw.items()) @@ -363,28 +364,42 @@ class EnvironmentValues(object): # # functionality # new_string_values.append(('', ValueTypes.STRING)) # new_parsed_values.append(None) + elif value.var_type == ValueTypes.COLLECTION: + # We may have received multiple values which may need further expansion. + # TODO: Not sure this is correct/best way. + # Do we need to maintain groups of collections for building command line - YES. + npv = [] + nsv = [] + for (t, v, i) in value.all_dependencies: + pv.lvars = pv.lvars.copy() + pv.lvars[pv.value] = '' + + if v == pv.value: + npv.append(None) + nsv.append(('', ValueTypes.STRING)) + else: + npv.append(ParsedEnvironmentValue(t, v, i, pv.lvars)) + nsv.append(None) + + new_parsed_values.extend(npv) + new_string_values.extend(nsv) else: - # TODO: Handle other recursive loops by empty stringing this value before recursing with - # copy of lvar? npv = [] nsv = [] for (t, v, i) in value.all_dependencies: + # blank out value in propagated lvars to prevent recursive variable loops pv.lvars = pv.lvars.copy() pv.lvars[pv.value] = '' if v == pv.value: npv.append(None) nsv.append(('', ValueTypes.STRING)) - # pv.lvars = pv.lvars.copy() - # pv.lvars[pv.value] = '' else: npv.append(ParsedEnvironmentValue(t, v, i, pv.lvars)) nsv.append(None) new_parsed_values.extend(npv) new_string_values.extend(nsv) - # new_parsed_values.extend([ParsedEnvironmentValue(t, v, i, lvars) for (t, v, i) in value.all_dependencies]) - # new_string_values.extend([None] * len(value.all_dependencies)) debug("%s->%s" % (pv.value, string_values[pv.position])) except KeyError as e: @@ -555,10 +570,10 @@ class EnvironmentValues(object): return s @staticmethod - def subst(substString, env, mode=0, target=None, source=None, gvars={}, lvars={}, conv=None): + def subst(subst_string, env, mode=0, target=None, source=None, gvars={}, lvars={}, conv=None): """ Recursively Expand string - :param substString: The string to be expanded. + :param subst_string: The string to be expanded. :param env: :param mode: 0 = leading or trailing white space will be removed from the result. and all sequences of white space will be compressed to a single space character. Additionally, any $( and $) character @@ -578,15 +593,15 @@ class EnvironmentValues(object): :return: expanded string """ # subst called with plain string, just return it. - if '$' not in substString: + if '$' not in subst_string: if mode != SubstModes.SUBST_LIST: - return substString + return subst_string else: - return [(substString, ValueTypes.STRING)] + return [(subst_string, ValueTypes.STRING)] - # TODO: Case to shortcut. substString = "$VAR" and env['VAR'] = simple string. + # TODO: Case to shortcut. subst_string = "$VAR" and env['VAR'] = simple string. # This happens enough to make it worth optimizing - # if ' ' not in substString and substString[0]=='$' and + # if ' ' not in subst_string and subst_string[0]=='$' and # TODO:Figure out how to handle this properly. May involve seeing if source & targets is specified. Or checking # against list of variables which are "ok" to leave undefined and unexpanded in the returned string and/or @@ -620,14 +635,14 @@ class EnvironmentValues(object): # being evaluated was previously at. try: - # First retrieve the value by substString and the applicable overrides. + # First retrieve the value by subst_string and the applicable overrides. # TODO: Transpose overrides set into something which is cacheable. String for now, but maybe tuple instead? override_string = "-".join(sorted(overrides)) - val = env.cached_values[(substString, override_string)] + val = env.cached_values[(subst_string, override_string)] except KeyError as e: # No such value create one - val = EnvironmentValue.factory(substString) + val = EnvironmentValue.factory(subst_string) string_values = [None] * len(val.all_dependencies) parsed_values = [None] * len(val.all_dependencies) @@ -667,12 +682,12 @@ class EnvironmentValues(object): return retval # try: - # var_type = env.values[substString].var_type + # var_type = env.values[subst_string].var_type # # if var_type == ValueTypes.STRING: - # return env.values[substString].value + # return env.values[subst_string].value # elif var_type == ValueTypes.PARSED: - # return env.values[substString].subst(env, mode=mode, target=target, source=source, gvars=gvars, + # return env.values[subst_string].subst(env, mode=mode, target=target, source=source, gvars=gvars, # lvars=lvars, conv=conv) # elif var_type == ValueTypes.CALLABLE: # # From Subst.py @@ -690,20 +705,20 @@ class EnvironmentValues(object): # # s = self.conv(s) # # return self.substitute(s, lvars) # # TODO: This can return a non-plain-string result which needs to be further processed. - # retval = env.values[substString].value(target, source, env, (mode == SubstModes.FOR_SIGNATURE)) + # retval = env.values[subst_string].value(target, source, env, (mode == SubstModes.FOR_SIGNATURE)) # return retval # except KeyError as e: - # if is_String(substString): + # if is_String(subst_string): # # The value requested to be substituted doesn't exist in the EnvironmentVariables. # # So, let's create a new value? # # Currently we're naming it the same as it's content. # # TODO: Should we keep these separate from the variables? We're caching both.. - # env.values[substString] = EnvironmentValue(substString) - # return env.values[substString].subst(env, mode=mode, target=target, source=source, gvars=gvars, + # env.values[subst_string] = EnvironmentValue(subst_string) + # return env.values[subst_string].subst(env, mode=mode, target=target, source=source, gvars=gvars, # lvars=lvars, conv=conv) - # elif is_Sequence(substString): + # elif is_Sequence(subst_string): # return [EnvironmentValue(v).subst(env, mode=mode, target=target, source=source, gvars=gvars, - # lvars=lvars, conv=conv) for v in substString] + # lvars=lvars, conv=conv) for v in subst_string] @staticmethod @@ -747,7 +762,9 @@ class EnvironmentValues(object): word.append(p) if word: - retval[line_no].append("".join(word)) + # retval[line_no].append("".join(word)) + retval[line_no].extend(word) + # if parts and len(parts) >= 1 and parts[0] != '': # or mode == SubstModes.RAW: # retval[line_no].append("".join(parts)) @@ -802,7 +819,7 @@ class EnvironmentValues(object): # listSubstVal = [i for i in listSubstVal if not i[0].isspace()] try: - debug("subst_list:IsString:%s", listSubstVal) + debug("subst_list:IsString:%s [%s]", listSubstVal, list_subst_list) except TypeError as e: debug("LKDJFKLSDJF") else: diff --git a/src/engine/SCons/EnvironmentValuesSubstTests.py b/src/engine/SCons/EnvironmentValuesSubstTests.py index 8dfbf2785..2a29a5265 100644 --- a/src/engine/SCons/EnvironmentValuesSubstTests.py +++ b/src/engine/SCons/EnvironmentValuesSubstTests.py @@ -1034,9 +1034,16 @@ class EnvVarsSubstListTestCase(SubstTestCase): # ["|", "a", "|", "b", "|", "c", "1"], # ["|", "|", "c", "1"], + # "test $($xxx$)", + # "test $($)", + + test = '${LIST}' - r = EnvironmentValues.subst_list(test, env, mode=SUBST_SIG, gvars=gvars) + # test = "test $($xxx$)" + r = EnvironmentValues.subst_list(test, env, mode=SUBST_RAW, gvars=gvars) answer = [['This', 'is', 'test']] + # answer = [["test", "$($)"]] + assert r == answer, 'This should be %s not :%s' % (answer, r) failed = 0 |