diff options
Diffstat (limited to 'src/buildstream/_variables.py')
| -rw-r--r-- | src/buildstream/_variables.py | 251 |
1 files changed, 251 insertions, 0 deletions
diff --git a/src/buildstream/_variables.py b/src/buildstream/_variables.py new file mode 100644 index 000000000..74314cf1f --- /dev/null +++ b/src/buildstream/_variables.py @@ -0,0 +1,251 @@ +# +# Copyright (C) 2016 Codethink Limited +# Copyright (C) 2019 Bloomberg L.P. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# Daniel Silverstone <daniel.silverstone@codethink.co.uk> + +import re +import sys + +from ._exceptions import LoadError, LoadErrorReason +from . import _yaml + +# Variables are allowed to have dashes here +# +PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}") + + +# Throughout this code you will see variables named things like `expstr`. +# These hold data structures called "expansion strings" and are the parsed +# form of the strings which are the input to this subsystem. Strings +# such as "Hello %{name}, how are you?" are parsed into the form: +# (3, ["Hello ", "name", ", how are you?"]) +# i.e. a tuple of an integer and a list, where the integer is the cached +# length of the list, and the list consists of one or more strings. +# Strings in even indices of the list (0, 2, 4, etc) are constants which +# are copied into the output of the expansion algorithm. Strings in the +# odd indices (1, 3, 5, etc) are the names of further expansions to make. +# In the example above, first "Hello " is copied, then "name" is expanded +# and so must be another named expansion string passed in to the constructor +# of the Variables class, and whatever is yielded from the expansion of "name" +# is added to the concatenation for the result. Finally ", how are you?" is +# copied in and the whole lot concatenated for return. +# +# To see how strings are parsed, see `_parse_expstr()` after the class, and +# to see how expansion strings are expanded, see `_expand_expstr()` after that. + + +# The Variables helper object will resolve the variable references in +# the given dictionary, expecting that any dictionary values which contain +# variable references can be resolved from the same dictionary. +# +# Each Element creates its own Variables instance to track the configured +# variable settings for the element. +# +# Args: +# node (dict): A node loaded and composited with yaml tools +# +# Raises: +# LoadError, if unresolved variables, or cycles in resolution, occur. +# +class Variables(): + + def __init__(self, node): + + self.original = node + self._expstr_map = self._resolve(node) + self.flat = self._flatten() + + # subst(): + # + # Substitutes any variables in 'string' and returns the result. + # + # Args: + # (string): The string to substitute + # + # Returns: + # (string): The new string with any substitutions made + # + # Raises: + # LoadError, if the string contains unresolved variable references. + # + def subst(self, string): + expstr = _parse_expstr(string) + + try: + return _expand_expstr(self._expstr_map, expstr) + except KeyError: + unmatched = [] + + # Look for any unmatched variable names in the expansion string + for var in expstr[1][1::2]: + if var not in self._expstr_map: + unmatched.append(var) + + if unmatched: + message = "Unresolved variable{}: {}".format( + "s" if len(unmatched) > 1 else "", + ", ".join(unmatched) + ) + + raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message) + # Otherwise, re-raise the KeyError since it clearly came from some + # other unknowable cause. + raise + + # Variable resolving code + # + # Here we resolve all of our inputs into a dictionary, ready for use + # in subst() + def _resolve(self, node): + # Special case, if notparallel is specified in the variables for this + # element, then override max-jobs to be 1. + # Initialize it as a string as all variables are processed as strings. + # + if _yaml.node_get(node, bool, 'notparallel', default_value=False): + _yaml.node_set(node, 'max-jobs', str(1)) + + ret = {} + for key, value in _yaml.node_items(node): + value = _yaml.node_get(node, str, key) + ret[sys.intern(key)] = _parse_expstr(value) + return ret + + def _check_for_missing(self): + # First the check for anything unresolvable + summary = [] + for key, expstr in self._expstr_map.items(): + for var in expstr[1][1::2]: + if var not in self._expstr_map: + line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}" + provenance = _yaml.node_get_provenance(self.original, key) + summary.append(line.format(unmatched=var, variable=key, provenance=provenance)) + if summary: + raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, + "Failed to resolve one or more variable:\n{}\n".format("\n".join(summary))) + + def _check_for_cycles(self): + # And now the cycle checks + def cycle_check(expstr, visited, cleared): + for var in expstr[1][1::2]: + if var in cleared: + continue + if var in visited: + raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE, + "{}: ".format(_yaml.node_get_provenance(self.original, var)) + + ("Variable '{}' expands to contain a reference to itself. " + + "Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var)) + visited.append(var) + cycle_check(self._expstr_map[var], visited, cleared) + visited.pop() + cleared.add(var) + + cleared = set() + for key, expstr in self._expstr_map.items(): + if key not in cleared: + cycle_check(expstr, [key], cleared) + + # _flatten(): + # + # Turn our dictionary of expansion strings into a flattened dict + # so that we can run expansions faster in the future + # + # Raises: + # LoadError, if the string contains unresolved variable references or + # if cycles are detected in the variable references + # + def _flatten(self): + flat = {} + try: + for key, expstr in self._expstr_map.items(): + if expstr[0] > 1: + expstr = (1, [sys.intern(_expand_expstr(self._expstr_map, expstr))]) + self._expstr_map[key] = expstr + flat[key] = expstr[1][0] + except KeyError: + self._check_for_missing() + raise + except RecursionError: + self._check_for_cycles() + raise + return flat + + +# Cache for the parsed expansion strings. While this is nominally +# something which might "waste" memory, in reality each of these +# will live as long as the element which uses it, which is the +# vast majority of the memory usage across the execution of BuildStream. +PARSE_CACHE = { + # Prime the cache with the empty string since otherwise that can + # cause issues with the parser, complications to which cause slowdown + "": (1, [""]), +} + + +# Helper to parse a string into an expansion string tuple, caching +# the results so that future parse requests don't need to think about +# the string +def _parse_expstr(instr): + try: + return PARSE_CACHE[instr] + except KeyError: + # This use of the regex turns a string like "foo %{bar} baz" into + # a list ["foo ", "bar", " baz"] + splits = PARSE_EXPANSION.split(instr) + # If an expansion ends the string, we get an empty string on the end + # which we can optimise away, making the expansion routines not need + # a test for this. + if splits[-1] == '': + splits = splits[:-1] + # Cache an interned copy of this. We intern it to try and reduce the + # memory impact of the cache. It seems odd to cache the list length + # but this is measurably cheaper than calculating it each time during + # string expansion. + PARSE_CACHE[instr] = (len(splits), [sys.intern(s) for s in splits]) + return PARSE_CACHE[instr] + + +# Helper to expand a given top level expansion string tuple in the context +# of the given dictionary of expansion strings. +# +# Note: Will raise KeyError if any expansion is missing +def _expand_expstr(content, topvalue): + # Short-circuit constant strings + if topvalue[0] == 1: + return topvalue[1][0] + + # Short-circuit strings which are entirely an expansion of another variable + # e.g. "%{another}" + if topvalue[0] == 2 and topvalue[1][0] == "": + return _expand_expstr(content, content[topvalue[1][1]]) + + # Otherwise process fully... + def internal_expand(value): + (expansion_len, expansion_bits) = value + idx = 0 + while idx < expansion_len: + # First yield any constant string content + yield expansion_bits[idx] + idx += 1 + # Now, if there is an expansion variable left to expand, yield + # the expansion of that variable too + if idx < expansion_len: + yield from internal_expand(content[expansion_bits[idx]]) + idx += 1 + + return "".join(internal_expand(topvalue)) |