_options: OptionPool implementation, core project options module

This sub package includes the OptionPool which is the buildstream
facing API for handling project options, loading them, evaluating
expressions based on options and pre-processing loaded YAML.

This also includes an abstract Option class and an implementation
for each supported option type.

8 files changed, 649 insertions, 0 deletions

diff --git a/buildstream/_options/__init__.py b/buildstream/_options/__init__.py
new file mode 100644
index 000000000..7b8f36553
--- /dev/null
+++ b/buildstream/_options/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .optionpool import OptionPool
diff --git a/buildstream/_options/option.py b/buildstream/_options/option.py
new file mode 100644
index 000000000..3accddccb
--- /dev/null
+++ b/buildstream/_options/option.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _yaml
+
+
+# Shared symbols for validation purposes
+#
+OPTION_SYMBOLS = [
+    'type',
+    'description'
+]
+
+
+# Option()
+#
+# An abstract class representing a project option.
+#
+# Concrete classes must be created to handle option types,
+# the loaded project options is a collection of typed Option
+# instances.
+#
+class Option():
+
+    # Subclasses use this to specify the type name used
+    # for the yaml format and error messages
+    OPTION_TYPE = None
+
+    def __init__(self, name, definition, pool):
+        self.name = name
+        self.description = None
+        self.value = None
+        self.pool = pool
+        self.load(definition)
+
+    # load()
+    #
+    # Loads the option attributes from the descriptions
+    # in the project.conf
+    #
+    # Args:
+    #    node (dict): The loaded YAML dictionary describing
+    #                 the option
+    def load(self, node):
+        self.description = _yaml.node_get(node, str, 'description')
+
+    # load_value()
+    #
+    # Loads the value of the option in string form.
+    #
+    # Args:
+    #    node (Mapping): The YAML loaded key/value dictionary
+    #                    to load the value from
+    #
+    def load_value(self, node):
+        pass  # pragma: nocover
+
+    # set_value()
+    #
+    # Sets the value of an option from a string passed
+    # to buildstream on the command line
+    #
+    # Args:
+    #    value (str): The value in string form
+    #
+    def set_value(self, value):
+        pass  # pragma: nocover
+
+    # resolve()
+    #
+    # Called on each option once, after all configuration
+    # and cli options have been passed.
+    #
+    def resolve(self):
+        pass  # pragma: nocover
diff --git a/buildstream/_options/optionarch.py b/buildstream/_options/optionarch.py
new file mode 100644
index 000000000..9cadbebba
--- /dev/null
+++ b/buildstream/_options/optionarch.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+import os
+from .option import OPTION_SYMBOLS
+from .optionenum import OptionEnum
+
+
+# OptionArch
+#
+# An enumeration project option which does not allow
+# definition of a default value, but instead tries to set
+# the default value to the machine architecture introspected
+# using `uname`
+#
+# Note that when using OptionArch in a project, it will automatically
+# bail out of the host machine `uname` reports a machine architecture
+# not supported by the project, in the case that no option was
+# specifically specified
+#
+class OptionArch(OptionEnum):
+
+    OPTION_TYPE = 'arch'
+
+    def load(self, node):
+        super(OptionArch, self).load(node, allow_default_definition=False)
+
+    def load_default_value(self, node):
+        _, _, _, _, machine_arch = os.uname()
+        return machine_arch
+
+    def resolve(self):
+
+        # Validate that the default machine arch reported by uname() is
+        # explicitly supported by the project, only if it was not
+        # overridden by user configuration or cli.
+        #
+        # If the value is specified on the cli or user configuration,
+        # then it will already be valid.
+        #
+        self.validate(self.value)
diff --git a/buildstream/_options/optionbool.py b/buildstream/_options/optionbool.py
new file mode 100644
index 000000000..d125e2d10
--- /dev/null
+++ b/buildstream/_options/optionbool.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _yaml
+from .. import LoadError, LoadErrorReason
+from .option import Option, OPTION_SYMBOLS
+
+
+# OptionBool
+#
+# A boolean project option
+#
+class OptionBool(Option):
+
+    OPTION_TYPE = 'bool'
+
+    def load(self, node):
+
+        super(OptionBool, self).load(node)
+        _yaml.node_validate(node, OPTION_SYMBOLS + ['default'])
+        self.value = _yaml.node_get(node, bool, 'default')
+
+    def load_value(self, node):
+        self.value = _yaml.node_get(node, bool, self.name)
+
+    def set_value(self, value):
+        if value == 'True' or value == 'true':
+            self.value = True
+        elif value == 'False' or value == 'false':
+            self.value = False
+        else:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "Invalid value for boolean option {}: {}".format(self.name, value))
diff --git a/buildstream/_options/optioneltmask.py b/buildstream/_options/optioneltmask.py
new file mode 100644
index 000000000..765c50d10
--- /dev/null
+++ b/buildstream/_options/optioneltmask.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import utils
+from .option import OPTION_SYMBOLS
+from .optionflags import OptionFlags
+
+
+# OptionEltMask
+#
+# A flags option which automatically only allows element
+# names as values.
+#
+class OptionEltMask(OptionFlags):
+
+    OPTION_TYPE = 'element-mask'
+
+    def load(self, node):
+        # Ask the parent constructor to disallow value definitions,
+        # we define those automatically only.
+        super(OptionEltMask, self).load(node, allow_value_definitions=False)
+
+    # Here we want all valid elements as possible values,
+    # but we'll settle for just the relative filenames
+    # of files ending with ".bst" in the project element directory
+    def load_valid_values(self, node):
+        values = []
+        for filename in utils.list_relative_paths(self.pool.element_path):
+            if filename.endswith('.bst'):
+                values.append(filename)
+        return values
diff --git a/buildstream/_options/optionenum.py b/buildstream/_options/optionenum.py
new file mode 100644
index 000000000..912c50ec3
--- /dev/null
+++ b/buildstream/_options/optionenum.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _yaml
+from .. import LoadError, LoadErrorReason
+from .option import Option, OPTION_SYMBOLS
+
+
+# OptionEnum
+#
+# An enumeration project option
+#
+class OptionEnum(Option):
+
+    OPTION_TYPE = 'enum'
+
+    def load(self, node, allow_default_definition=True):
+        super(OptionEnum, self).load(node)
+
+        valid_symbols = OPTION_SYMBOLS + ['values']
+        if allow_default_definition:
+            valid_symbols += ['default']
+
+        _yaml.node_validate(node, valid_symbols)
+
+        self.values = _yaml.node_get(node, list, 'values', default_value=[])
+        if not self.values:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: No values specified for {} option '{}'"
+                            .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name))
+
+        # Allow subclass to define the default value
+        self.value = self.load_default_value(node)
+
+    def load_value(self, node):
+        self.value = _yaml.node_get(node, str, self.name)
+        self.validate(self.value, _yaml.node_get_provenance(node, self.name))
+
+    def set_value(self, value):
+        self.validate(value)
+        self.value = value
+
+    def validate(self, value, provenance=None):
+        if value not in self.values:
+            prefix = ""
+            if provenance:
+                prefix = "{}: ".format(provenance)
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}Invalid value for {} option '{}': {}\n"
+                            .format(prefix, self.OPTION_TYPE, self.name, value) +
+                            "Valid values: {}".format(", ".join(self.values)))
+
+    def load_default_value(self, node):
+        value = _yaml.node_get(node, str, 'default')
+        self.validate(value, _yaml.node_get_provenance(node, 'default'))
+        return value
diff --git a/buildstream/_options/optionflags.py b/buildstream/_options/optionflags.py
new file mode 100644
index 000000000..7abb85786
--- /dev/null
+++ b/buildstream/_options/optionflags.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _yaml
+from .. import LoadError, LoadErrorReason
+from .option import Option, OPTION_SYMBOLS
+
+
+# OptionFlags
+#
+# A flags project option
+#
+class OptionFlags(Option):
+
+    OPTION_TYPE = 'flags'
+
+    def load(self, node, allow_value_definitions=True):
+        super(OptionFlags, self).load(node)
+
+        valid_symbols = OPTION_SYMBOLS + ['default']
+        if allow_value_definitions:
+            valid_symbols += ['values']
+
+        _yaml.node_validate(node, valid_symbols)
+
+        # Allow subclass to define the valid values
+        self.values = self.load_valid_values(node)
+        if not self.values:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: No values specified for {} option '{}'"
+                            .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name))
+
+        self.value = _yaml.node_get(node, list, 'default', default_value=[])
+        self.validate(self.value, _yaml.node_get_provenance(node, 'default'))
+
+    def load_value(self, node):
+        self.value = _yaml.node_get(node, list, self.name)
+        self.value = sorted(self.value)
+        self.validate(self.value, _yaml.node_get_provenance(node, self.name))
+
+    def set_value(self, value):
+        # Strip out all whitespace, allowing: "value1, value2 , value3"
+        stripped = "".join(value.split())
+
+        # Get the comma separated values
+        list_value = stripped.split(',')
+
+        self.validate(list_value)
+        self.value = sorted(list_value)
+
+    def validate(self, value, provenance=None):
+        for flag in value:
+            if flag not in self.values:
+                prefix = ""
+                if provenance:
+                    prefix = "{}: ".format(provenance)
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}Invalid value for flags option '{}': {}\n"
+                                .format(prefix, self.name, value) +
+                                "Valid values: {}".format(", ".join(self.values)))
+
+    def load_valid_values(self, node):
+        # Allow the more descriptive error to raise when no values
+        # exist rather than bailing out here (by specifying default_value)
+        return _yaml.node_get(node, list, 'values', default_value=[])
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
new file mode 100644
index 000000000..3686ee6e3
--- /dev/null
+++ b/buildstream/_options/optionpool.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#
+import ast
+import jinja2
+from collections import Mapping
+
+from .. import _yaml
+from .. import LoadError, LoadErrorReason
+from .optionbool import OptionBool
+from .optionenum import OptionEnum
+from .optionflags import OptionFlags
+from .optioneltmask import OptionEltMask
+from .optionarch import OptionArch
+
+
+OPTION_TYPES = {
+    OptionBool.OPTION_TYPE: OptionBool,
+    OptionEnum.OPTION_TYPE: OptionEnum,
+    OptionFlags.OPTION_TYPE: OptionFlags,
+    OptionEltMask.OPTION_TYPE: OptionEltMask,
+    OptionArch.OPTION_TYPE: OptionArch,
+}
+
+
+class OptionPool():
+
+    def __init__(self, element_path):
+        self.options = {}      # The Options
+        self.variables = None  # The Options resolved into typed variables
+
+        # We hold on to the element path for the sake of OptionEltMask
+        self.element_path = element_path
+
+        # jinja2 environment, with default globals cleared out of the way
+        self.environment = jinja2.Environment()
+        self.environment.globals = []
+
+    # load()
+    #
+    # Loads the options described in the project.conf
+    #
+    # Args:
+    #    node (dict): The loaded YAML options
+    #
+    def load(self, options):
+
+        for option_name, option_definition in _yaml.node_items(options):
+            opt_type_name = _yaml.node_get(option_definition, str, 'type')
+
+            try:
+                opt_type = OPTION_TYPES[opt_type_name]
+            except KeyError:
+                p = _yaml.node_get_provenance(option_definition, 'type')
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}: Invalid option type '{}'".format(p, opt_type_name))
+
+            option = opt_type(option_name, option_definition, self)
+            self.options[option_name] = option
+
+    # load_values()
+    #
+    # Loads the option values specified in a key/value
+    # dictionary loaded from YAML, and a list of tuples
+    # collected from the command line
+    #
+    # Args:
+    #    node (dict): The loaded YAML options
+    #    cli_options (list): A list of (str, str) tuples
+    #
+    def load_values(self, node, cli_options):
+        for option_name, _ in _yaml.node_items(node):
+            try:
+                option = self.options[option_name]
+            except KeyError as e:
+                p = _yaml.node_get_provenance(node, option_name)
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "{}: Unknown option '{}' specified".format(p, option_name))
+            option.load_value(node)
+
+        for option_name, option_value in cli_options:
+            try:
+                option = self.options[option_name]
+            except KeyError as e:
+                raise LoadError(LoadErrorReason.INVALID_DATA,
+                                "Unknown option '{}' specified on the command line".format(option_name))
+            option.set_value(option_value)
+
+    # resolve()
+    #
+    # Resolves the loaded options, this is just a step which must be
+    # performed after loading all options and their values, and before
+    # ever trying to evaluate an expression
+    #
+    def resolve(self):
+        self.variables = {}
+        for option_name, option in self.options.items():
+            # Delegate one more method for options to
+            # do some last minute validation once any
+            # overrides have been performed.
+            #
+            option.resolve()
+
+            self.variables[option_name] = option.value
+
+    # evaluate()
+    #
+    # Evaluates a jinja2 style expression with the loaded options in context.
+    #
+    # Args:
+    #    expression (str): The jinja2 style expression
+    #
+    # Returns:
+    #    (bool): Whether the expression resolved to a truthy value or a falsy one.
+    #
+    # Raises:
+    #    LoadError: If the expression failed to resolve for any reason
+    #
+    def evaluate(self, expression):
+
+        #
+        # Variables must be resolved at this point.
+        #
+        try:
+            template_string = "{%% if %s %%} True {%% else %%} False {%% endif %%}" % expression
+            template = self.environment.from_string(template_string)
+            context = template.new_context(self.variables, shared=True)
+            result = template.root_render_func(context)
+            evaluated = jinja2.utils.concat(result)
+            val = evaluated.strip()
+
+            if val == "True":
+                return True
+            elif val == "False":
+                return False
+            else:  # pragma: nocover
+                raise LoadError(LoadErrorReason.EXPRESSION_FAILED,
+                                "Failed to evaluate expression: {}".format(expression))
+        except jinja2.exceptions.TemplateError as e:
+            raise LoadError(LoadErrorReason.EXPRESSION_FAILED,
+                            "Failed to evaluate expression ({}): {}".format(expression, e))
+
+    # process_node()
+    #
+    # Args:
+    #    node (Mapping): A YAML Loaded dictionary
+    #
+    def process_node(self, node):
+
+        # A conditional will result in composition, which can
+        # in turn add new conditionals to the root.
+        #
+        # Keep processing conditionals on the root node until
+        # all directly nested conditionals are resolved.
+        #
+        while self.process_conditional(node):
+            pass
+
+        # Now recurse into nested dictionaries and lists
+        # and process any indirectly nested conditionals.
+        #
+        for key, value in _yaml.node_items(node):
+            if isinstance(value, Mapping):
+                self.process_node(value)
+            elif isinstance(value, list):
+                self.process_list(value)
+
+    # Recursion assistent for lists, in case there
+    # are lists of lists.
+    #
+    def process_list(self, values):
+        for value in values:
+            if isinstance(value, Mapping):
+                self.process_node(value)
+            elif isinstance(value, list):
+                self.process_list(value)
+
+    # Process a single conditional, resulting in composition
+    # at the root level on the passed node
+    #
+    # Return true if a conditional was processed.
+    #
+    def process_conditional(self, node):
+        conditions = _yaml.node_get(node, list, '(?)', default_value=[]) or None
+
+        if conditions is not None:
+
+            # Collect provenance first, we need to delete the (?) key
+            # before any composition occurs.
+            provenance = [
+                _yaml.node_get_provenance(node, '(?)', indices=[i])
+                for i in range(len(conditions))
+            ]
+            del node['(?)']
+
+            for condition, p in zip(conditions, provenance):
+                tuples = list(_yaml.node_items(condition))
+                if len(tuples) > 1:
+                    raise LoadError(LoadErrorReason.INVALID_DATA,
+                                    "{}: Conditional statement has more than one key".format(p))
+
+                expression, value = tuples[0]
+                if self.evaluate(expression):
+                    _yaml.composite_dict(node, value)
+
+            return True
+
+        return False