diff options
-rw-r--r-- | buildstream/_options/__init__.py | 21 | ||||
-rw-r--r-- | buildstream/_options/option.py | 92 | ||||
-rw-r--r-- | buildstream/_options/optionarch.py | 58 | ||||
-rw-r--r-- | buildstream/_options/optionbool.py | 50 | ||||
-rw-r--r-- | buildstream/_options/optioneltmask.py | 48 | ||||
-rw-r--r-- | buildstream/_options/optionenum.py | 73 | ||||
-rw-r--r-- | buildstream/_options/optionflags.py | 82 | ||||
-rw-r--r-- | buildstream/_options/optionpool.py | 225 |
8 files changed, 649 insertions, 0 deletions
diff --git a/buildstream/_options/__init__.py b/buildstream/_options/__init__.py new file mode 100644 index 000000000..7b8f36553 --- /dev/null +++ b/buildstream/_options/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .optionpool import OptionPool diff --git a/buildstream/_options/option.py b/buildstream/_options/option.py new file mode 100644 index 000000000..3accddccb --- /dev/null +++ b/buildstream/_options/option.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .. import _yaml + + +# Shared symbols for validation purposes +# +OPTION_SYMBOLS = [ + 'type', + 'description' +] + + +# Option() +# +# An abstract class representing a project option. +# +# Concrete classes must be created to handle option types, +# the loaded project options is a collection of typed Option +# instances. +# +class Option(): + + # Subclasses use this to specify the type name used + # for the yaml format and error messages + OPTION_TYPE = None + + def __init__(self, name, definition, pool): + self.name = name + self.description = None + self.value = None + self.pool = pool + self.load(definition) + + # load() + # + # Loads the option attributes from the descriptions + # in the project.conf + # + # Args: + # node (dict): The loaded YAML dictionary describing + # the option + def load(self, node): + self.description = _yaml.node_get(node, str, 'description') + + # load_value() + # + # Loads the value of the option in string form. + # + # Args: + # node (Mapping): The YAML loaded key/value dictionary + # to load the value from + # + def load_value(self, node): + pass # pragma: nocover + + # set_value() + # + # Sets the value of an option from a string passed + # to buildstream on the command line + # + # Args: + # value (str): The value in string form + # + def set_value(self, value): + pass # pragma: nocover + + # resolve() + # + # Called on each option once, after all configuration + # and cli options have been passed. + # + def resolve(self): + pass # pragma: nocover diff --git a/buildstream/_options/optionarch.py b/buildstream/_options/optionarch.py new file mode 100644 index 000000000..9cadbebba --- /dev/null +++ b/buildstream/_options/optionarch.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +import os +from .option import OPTION_SYMBOLS +from .optionenum import OptionEnum + + +# OptionArch +# +# An enumeration project option which does not allow +# definition of a default value, but instead tries to set +# the default value to the machine architecture introspected +# using `uname` +# +# Note that when using OptionArch in a project, it will automatically +# bail out of the host machine `uname` reports a machine architecture +# not supported by the project, in the case that no option was +# specifically specified +# +class OptionArch(OptionEnum): + + OPTION_TYPE = 'arch' + + def load(self, node): + super(OptionArch, self).load(node, allow_default_definition=False) + + def load_default_value(self, node): + _, _, _, _, machine_arch = os.uname() + return machine_arch + + def resolve(self): + + # Validate that the default machine arch reported by uname() is + # explicitly supported by the project, only if it was not + # overridden by user configuration or cli. + # + # If the value is specified on the cli or user configuration, + # then it will already be valid. + # + self.validate(self.value) diff --git a/buildstream/_options/optionbool.py b/buildstream/_options/optionbool.py new file mode 100644 index 000000000..d125e2d10 --- /dev/null +++ b/buildstream/_options/optionbool.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .. import _yaml +from .. import LoadError, LoadErrorReason +from .option import Option, OPTION_SYMBOLS + + +# OptionBool +# +# A boolean project option +# +class OptionBool(Option): + + OPTION_TYPE = 'bool' + + def load(self, node): + + super(OptionBool, self).load(node) + _yaml.node_validate(node, OPTION_SYMBOLS + ['default']) + self.value = _yaml.node_get(node, bool, 'default') + + def load_value(self, node): + self.value = _yaml.node_get(node, bool, self.name) + + def set_value(self, value): + if value == 'True' or value == 'true': + self.value = True + elif value == 'False' or value == 'false': + self.value = False + else: + raise LoadError(LoadErrorReason.INVALID_DATA, + "Invalid value for boolean option {}: {}".format(self.name, value)) diff --git a/buildstream/_options/optioneltmask.py b/buildstream/_options/optioneltmask.py new file mode 100644 index 000000000..765c50d10 --- /dev/null +++ b/buildstream/_options/optioneltmask.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .. import utils +from .option import OPTION_SYMBOLS +from .optionflags import OptionFlags + + +# OptionEltMask +# +# A flags option which automatically only allows element +# names as values. +# +class OptionEltMask(OptionFlags): + + OPTION_TYPE = 'element-mask' + + def load(self, node): + # Ask the parent constructor to disallow value definitions, + # we define those automatically only. + super(OptionEltMask, self).load(node, allow_value_definitions=False) + + # Here we want all valid elements as possible values, + # but we'll settle for just the relative filenames + # of files ending with ".bst" in the project element directory + def load_valid_values(self, node): + values = [] + for filename in utils.list_relative_paths(self.pool.element_path): + if filename.endswith('.bst'): + values.append(filename) + return values diff --git a/buildstream/_options/optionenum.py b/buildstream/_options/optionenum.py new file mode 100644 index 000000000..912c50ec3 --- /dev/null +++ b/buildstream/_options/optionenum.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .. import _yaml +from .. import LoadError, LoadErrorReason +from .option import Option, OPTION_SYMBOLS + + +# OptionEnum +# +# An enumeration project option +# +class OptionEnum(Option): + + OPTION_TYPE = 'enum' + + def load(self, node, allow_default_definition=True): + super(OptionEnum, self).load(node) + + valid_symbols = OPTION_SYMBOLS + ['values'] + if allow_default_definition: + valid_symbols += ['default'] + + _yaml.node_validate(node, valid_symbols) + + self.values = _yaml.node_get(node, list, 'values', default_value=[]) + if not self.values: + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: No values specified for {} option '{}'" + .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name)) + + # Allow subclass to define the default value + self.value = self.load_default_value(node) + + def load_value(self, node): + self.value = _yaml.node_get(node, str, self.name) + self.validate(self.value, _yaml.node_get_provenance(node, self.name)) + + def set_value(self, value): + self.validate(value) + self.value = value + + def validate(self, value, provenance=None): + if value not in self.values: + prefix = "" + if provenance: + prefix = "{}: ".format(provenance) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}Invalid value for {} option '{}': {}\n" + .format(prefix, self.OPTION_TYPE, self.name, value) + + "Valid values: {}".format(", ".join(self.values))) + + def load_default_value(self, node): + value = _yaml.node_get(node, str, 'default') + self.validate(value, _yaml.node_get_provenance(node, 'default')) + return value diff --git a/buildstream/_options/optionflags.py b/buildstream/_options/optionflags.py new file mode 100644 index 000000000..7abb85786 --- /dev/null +++ b/buildstream/_options/optionflags.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .. import _yaml +from .. import LoadError, LoadErrorReason +from .option import Option, OPTION_SYMBOLS + + +# OptionFlags +# +# A flags project option +# +class OptionFlags(Option): + + OPTION_TYPE = 'flags' + + def load(self, node, allow_value_definitions=True): + super(OptionFlags, self).load(node) + + valid_symbols = OPTION_SYMBOLS + ['default'] + if allow_value_definitions: + valid_symbols += ['values'] + + _yaml.node_validate(node, valid_symbols) + + # Allow subclass to define the valid values + self.values = self.load_valid_values(node) + if not self.values: + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: No values specified for {} option '{}'" + .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name)) + + self.value = _yaml.node_get(node, list, 'default', default_value=[]) + self.validate(self.value, _yaml.node_get_provenance(node, 'default')) + + def load_value(self, node): + self.value = _yaml.node_get(node, list, self.name) + self.value = sorted(self.value) + self.validate(self.value, _yaml.node_get_provenance(node, self.name)) + + def set_value(self, value): + # Strip out all whitespace, allowing: "value1, value2 , value3" + stripped = "".join(value.split()) + + # Get the comma separated values + list_value = stripped.split(',') + + self.validate(list_value) + self.value = sorted(list_value) + + def validate(self, value, provenance=None): + for flag in value: + if flag not in self.values: + prefix = "" + if provenance: + prefix = "{}: ".format(provenance) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}Invalid value for flags option '{}': {}\n" + .format(prefix, self.name, value) + + "Valid values: {}".format(", ".join(self.values))) + + def load_valid_values(self, node): + # Allow the more descriptive error to raise when no values + # exist rather than bailing out here (by specifying default_value) + return _yaml.node_get(node, list, 'values', default_value=[]) diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py new file mode 100644 index 000000000..3686ee6e3 --- /dev/null +++ b/buildstream/_options/optionpool.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# +import ast +import jinja2 +from collections import Mapping + +from .. import _yaml +from .. import LoadError, LoadErrorReason +from .optionbool import OptionBool +from .optionenum import OptionEnum +from .optionflags import OptionFlags +from .optioneltmask import OptionEltMask +from .optionarch import OptionArch + + +OPTION_TYPES = { + OptionBool.OPTION_TYPE: OptionBool, + OptionEnum.OPTION_TYPE: OptionEnum, + OptionFlags.OPTION_TYPE: OptionFlags, + OptionEltMask.OPTION_TYPE: OptionEltMask, + OptionArch.OPTION_TYPE: OptionArch, +} + + +class OptionPool(): + + def __init__(self, element_path): + self.options = {} # The Options + self.variables = None # The Options resolved into typed variables + + # We hold on to the element path for the sake of OptionEltMask + self.element_path = element_path + + # jinja2 environment, with default globals cleared out of the way + self.environment = jinja2.Environment() + self.environment.globals = [] + + # load() + # + # Loads the options described in the project.conf + # + # Args: + # node (dict): The loaded YAML options + # + def load(self, options): + + for option_name, option_definition in _yaml.node_items(options): + opt_type_name = _yaml.node_get(option_definition, str, 'type') + + try: + opt_type = OPTION_TYPES[opt_type_name] + except KeyError: + p = _yaml.node_get_provenance(option_definition, 'type') + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Invalid option type '{}'".format(p, opt_type_name)) + + option = opt_type(option_name, option_definition, self) + self.options[option_name] = option + + # load_values() + # + # Loads the option values specified in a key/value + # dictionary loaded from YAML, and a list of tuples + # collected from the command line + # + # Args: + # node (dict): The loaded YAML options + # cli_options (list): A list of (str, str) tuples + # + def load_values(self, node, cli_options): + for option_name, _ in _yaml.node_items(node): + try: + option = self.options[option_name] + except KeyError as e: + p = _yaml.node_get_provenance(node, option_name) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Unknown option '{}' specified".format(p, option_name)) + option.load_value(node) + + for option_name, option_value in cli_options: + try: + option = self.options[option_name] + except KeyError as e: + raise LoadError(LoadErrorReason.INVALID_DATA, + "Unknown option '{}' specified on the command line".format(option_name)) + option.set_value(option_value) + + # resolve() + # + # Resolves the loaded options, this is just a step which must be + # performed after loading all options and their values, and before + # ever trying to evaluate an expression + # + def resolve(self): + self.variables = {} + for option_name, option in self.options.items(): + # Delegate one more method for options to + # do some last minute validation once any + # overrides have been performed. + # + option.resolve() + + self.variables[option_name] = option.value + + # evaluate() + # + # Evaluates a jinja2 style expression with the loaded options in context. + # + # Args: + # expression (str): The jinja2 style expression + # + # Returns: + # (bool): Whether the expression resolved to a truthy value or a falsy one. + # + # Raises: + # LoadError: If the expression failed to resolve for any reason + # + def evaluate(self, expression): + + # + # Variables must be resolved at this point. + # + try: + template_string = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % expression + template = self.environment.from_string(template_string) + context = template.new_context(self.variables, shared=True) + result = template.root_render_func(context) + evaluated = jinja2.utils.concat(result) + val = evaluated.strip() + + if val == "True": + return True + elif val == "False": + return False + else: # pragma: nocover + raise LoadError(LoadErrorReason.EXPRESSION_FAILED, + "Failed to evaluate expression: {}".format(expression)) + except jinja2.exceptions.TemplateError as e: + raise LoadError(LoadErrorReason.EXPRESSION_FAILED, + "Failed to evaluate expression ({}): {}".format(expression, e)) + + # process_node() + # + # Args: + # node (Mapping): A YAML Loaded dictionary + # + def process_node(self, node): + + # A conditional will result in composition, which can + # in turn add new conditionals to the root. + # + # Keep processing conditionals on the root node until + # all directly nested conditionals are resolved. + # + while self.process_conditional(node): + pass + + # Now recurse into nested dictionaries and lists + # and process any indirectly nested conditionals. + # + for key, value in _yaml.node_items(node): + if isinstance(value, Mapping): + self.process_node(value) + elif isinstance(value, list): + self.process_list(value) + + # Recursion assistent for lists, in case there + # are lists of lists. + # + def process_list(self, values): + for value in values: + if isinstance(value, Mapping): + self.process_node(value) + elif isinstance(value, list): + self.process_list(value) + + # Process a single conditional, resulting in composition + # at the root level on the passed node + # + # Return true if a conditional was processed. + # + def process_conditional(self, node): + conditions = _yaml.node_get(node, list, '(?)', default_value=[]) or None + + if conditions is not None: + + # Collect provenance first, we need to delete the (?) key + # before any composition occurs. + provenance = [ + _yaml.node_get_provenance(node, '(?)', indices=[i]) + for i in range(len(conditions)) + ] + del node['(?)'] + + for condition, p in zip(conditions, provenance): + tuples = list(_yaml.node_items(condition)) + if len(tuples) > 1: + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Conditional statement has more than one key".format(p)) + + expression, value = tuples[0] + if self.evaluate(expression): + _yaml.composite_dict(node, value) + + return True + + return False |