diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/buildstream/_loader/loadelement.pyx | 35 | ||||
| -rw-r--r-- | src/buildstream/_loader/loader.py | 201 | ||||
| -rw-r--r-- | src/buildstream/exceptions.py | 3 | ||||
| -rw-r--r-- | src/buildstream/plugins/elements/junction.py | 62 | ||||
| -rw-r--r-- | src/buildstream/plugins/elements/link.py | 90 |
5 files changed, 251 insertions, 140 deletions
diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx index 31c3aef1a..2f4e7a0f9 100644 --- a/src/buildstream/_loader/loadelement.pyx +++ b/src/buildstream/_loader/loadelement.pyx @@ -21,8 +21,11 @@ from functools import cmp_to_key from pyroaring import BitMap, FrozenBitMap # pylint: disable=no-name-in-module -from ..node cimport MappingNode -from .types import Symbol +from .._exceptions import LoadError +from ..exceptions import LoadErrorReason +from ..element import Element +from ..node cimport MappingNode, ProvenanceInformation +from .types import Symbol, extract_depends_from_node # Counter to get ids to LoadElements @@ -74,6 +77,8 @@ cdef class LoadElement: cdef public bint meta_done cdef int node_id cdef readonly object _loader + cdef readonly str link_target + cdef readonly ProvenanceInformation link_target_provenance # TODO: if/when pyroaring exports symbols, we could type this statically cdef object _dep_cache cdef readonly list dependencies @@ -88,6 +93,8 @@ cdef class LoadElement: self.full_name = None # The element full name (with associated junction) self.meta_done = False # If the MetaElement for this LoadElement is done self.node_id = _next_synthetic_counter() + self.link_target = None # The target of a link element + self.link_target_provenance = None # The provenance of the link target # # Private members @@ -105,6 +112,8 @@ cdef class LoadElement: # dependency is in top-level project self.full_name = self.name + self.dependencies = [] + # Ensure the root node is valid self.node.validate_keys([ 'kind', 'depends', 'sources', 'sandbox', @@ -113,7 +122,27 @@ cdef class LoadElement: 'build-depends', 'runtime-depends', ]) - self.dependencies = [] + # + # If this is a link, resolve it right away and just + # store the link target and provenance + # + if self.node.get_str(Symbol.KIND, default=None) == 'link': + meta_element = self._loader.collect_element_no_deps(self, None) + element = Element._new_from_meta(meta_element) + element._initialize_state() + + # Custom error for link dependencies, since we don't completely + # parse their dependencies we cannot rely on the built-in ElementError. + deps = extract_depends_from_node(self.node) + if deps: + provenance = self.node + raise LoadError( + "{}: Dependencies are forbidden for 'link' elements".format(element), + LoadErrorReason.LINK_FORBIDDEN_DEPENDENCIES + ) + + self.link_target = element.target + self.link_target_provenance = element.target_provenance @property def junction(self): diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 49018629e..b73f5b862 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -217,6 +217,27 @@ class Loader: self._loaders[filename] = None return None + # At this point we've loaded the LoadElement + load_element = self._elements[filename] + + # If the loaded element is a link, then just follow it + # immediately and move on to the target. + # + if load_element.link_target: + + _, filename, loader = self._parse_name(load_element.link_target, rewritable, ticker) + + if loader != self: + level = level + 1 + + return loader.get_loader( + filename, + rewritable=rewritable, + ticker=ticker, + level=level, + provenance=load_element.link_target_provenance, + ) + # meta junction element # # Note that junction elements are not allowed to have @@ -227,7 +248,7 @@ class Loader: # # Any task counting *inside* the junction will be handled by # its loader. - meta_element = self._collect_element_no_deps(self._elements[filename]) + meta_element = self.collect_element_no_deps(self._elements[filename]) if meta_element.kind != "junction": raise LoadError( "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind), @@ -247,24 +268,12 @@ class Loader: # through the entire loading process), which is nice UX. It # would be nice if this could be done for *all* element types, # but since we haven't loaded those yet that's impossible. - if self._elements[filename].dependencies: + if load_element.dependencies: raise LoadError("Dependencies are forbidden for 'junction' elements", LoadErrorReason.INVALID_JUNCTION) element = Element._new_from_meta(meta_element) element._initialize_state() - # If this junction element points to a sub-sub-project, we need to - # find loader for that project. - if element.target: - subproject_loader = self.get_loader( - element.target_junction, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance - ) - loader = subproject_loader.get_loader( - element.target_element, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance - ) - self._loaders[filename] = loader - return loader - # Handle the case where a subproject has no ref # if not element._has_all_sources_resolved(): @@ -368,6 +377,81 @@ class Loader: return state + # collect_element_no_deps() + # + # Collect a single element, without its dependencies, into a meta_element + # + # NOTE: This is declared public in order to share with the LoadElement + # and should not be used outside of that `_loader` module. + # + # Args: + # element (LoadElement): The element for which to load a MetaElement + # task (Task): A task to write progress information to + # + # Returns: + # (MetaElement): A partially loaded MetaElement + # + def collect_element_no_deps(self, element, task=None): + # Return the already built one, if we already built it + meta_element = self._meta_elements.get(element.name) + if meta_element: + return meta_element + + node = element.node + elt_provenance = node.get_provenance() + meta_sources = [] + + element_kind = node.get_str(Symbol.KIND) + + # if there's a workspace for this element then just append a dummy workspace + # metasource. + workspace = self._context.get_workspaces().get_workspace(element.name) + skip_workspace = True + if workspace: + workspace_node = {"kind": "workspace"} + workspace_node["path"] = workspace.get_absolute_path() + workspace_node["last_build"] = str(workspace.to_dict().get("last_build", "")) + node[Symbol.SOURCES] = [workspace_node] + skip_workspace = False + + sources = node.get_sequence(Symbol.SOURCES, default=[]) + for index, source in enumerate(sources): + kind = source.get_str(Symbol.KIND) + # the workspace source plugin cannot be used unless the element is workspaced + if kind == "workspace" and skip_workspace: + continue + + del source[Symbol.KIND] + + # Directory is optional + directory = source.get_str(Symbol.DIRECTORY, default=None) + if directory: + del source[Symbol.DIRECTORY] + meta_source = MetaSource(element.name, index, element_kind, kind, source, directory) + meta_sources.append(meta_source) + + meta_element = MetaElement( + self.project, + element.name, + element_kind, + elt_provenance, + meta_sources, + node.get_mapping(Symbol.CONFIG, default={}), + node.get_mapping(Symbol.VARIABLES, default={}), + node.get_mapping(Symbol.ENVIRONMENT, default={}), + node.get_str_list(Symbol.ENV_NOCACHE, default=[]), + node.get_mapping(Symbol.PUBLIC, default={}), + node.get_mapping(Symbol.SANDBOX, default={}), + element_kind == "junction", + ) + + # Cache it now, make sure it's already there before recursing + self._meta_elements[element.name] = meta_element + if task: + task.add_current_progress() + + return meta_element + ########################################### # Private Methods # ########################################### @@ -474,6 +558,13 @@ class Loader: ticker(filename) top_element = self._load_file_no_deps(filename, rewritable, provenance) + + # If this element is a link then we need to resolve it + # and replace the dependency we've processed with this one + if top_element.link_target is not None: + _, filename, loader = self._parse_name(top_element.link_target, rewritable, ticker) + top_element = loader._load_file(filename, rewritable, ticker, top_element.link_target_provenance) + dependencies = extract_depends_from_node(top_element.node) # The loader queue is a stack of tuples # [0] is the LoadElement instance @@ -513,6 +604,12 @@ class Loader: "{}: Cannot depend on junction".format(dep.provenance), LoadErrorReason.INVALID_DATA ) + # If this dependency is a link then we need to resolve it + # and replace the dependency we've processed with this one + if dep_element.link_target: + _, filename, loader = self._parse_name(dep_element.link_target, rewritable, ticker) + dep_element = loader._load_file(filename, rewritable, ticker, dep_element.link_target_provenance) + # All is well, push the dependency onto the LoadElement # Pylint is not very happy with Cython and can't understand 'dependencies' is a list current_element[0].dependencies.append( # pylint: disable=no-member @@ -581,78 +678,6 @@ class Loader: check_elements.remove(this_element) validated.add(this_element) - # _collect_element_no_deps() - # - # Collect a single element, without its dependencies, into a meta_element - # - # Args: - # element (LoadElement): The element for which to load a MetaElement - # task (Task): A task to write progress information to - # - # Returns: - # (MetaElement): A partially loaded MetaElement - # - def _collect_element_no_deps(self, element, task=None): - # Return the already built one, if we already built it - meta_element = self._meta_elements.get(element.name) - if meta_element: - return meta_element - - node = element.node - elt_provenance = node.get_provenance() - meta_sources = [] - - element_kind = node.get_str(Symbol.KIND) - - # if there's a workspace for this element then just append a dummy workspace - # metasource. - workspace = self._context.get_workspaces().get_workspace(element.name) - skip_workspace = True - if workspace: - workspace_node = {"kind": "workspace"} - workspace_node["path"] = workspace.get_absolute_path() - workspace_node["last_build"] = str(workspace.to_dict().get("last_build", "")) - node[Symbol.SOURCES] = [workspace_node] - skip_workspace = False - - sources = node.get_sequence(Symbol.SOURCES, default=[]) - for index, source in enumerate(sources): - kind = source.get_str(Symbol.KIND) - # the workspace source plugin cannot be used unless the element is workspaced - if kind == "workspace" and skip_workspace: - continue - - del source[Symbol.KIND] - - # Directory is optional - directory = source.get_str(Symbol.DIRECTORY, default=None) - if directory: - del source[Symbol.DIRECTORY] - meta_source = MetaSource(element.name, index, element_kind, kind, source, directory) - meta_sources.append(meta_source) - - meta_element = MetaElement( - self.project, - element.name, - element_kind, - elt_provenance, - meta_sources, - node.get_mapping(Symbol.CONFIG, default={}), - node.get_mapping(Symbol.VARIABLES, default={}), - node.get_mapping(Symbol.ENVIRONMENT, default={}), - node.get_str_list(Symbol.ENV_NOCACHE, default=[]), - node.get_mapping(Symbol.PUBLIC, default={}), - node.get_mapping(Symbol.SANDBOX, default={}), - element_kind == "junction", - ) - - # Cache it now, make sure it's already there before recursing - self._meta_elements[element.name] = meta_element - if task: - task.add_current_progress() - - return meta_element - # _collect_element() # # Collect the toplevel elements we have @@ -666,7 +691,7 @@ class Loader: # def _collect_element(self, top_element, task=None): element_queue = [top_element] - meta_element_queue = [self._collect_element_no_deps(top_element, task)] + meta_element_queue = [self.collect_element_no_deps(top_element, task)] while element_queue: element = element_queue.pop() @@ -683,7 +708,7 @@ class Loader: name = dep.element.name if name not in loader._meta_elements: - meta_dep = loader._collect_element_no_deps(dep.element, task) + meta_dep = loader.collect_element_no_deps(dep.element, task) element_queue.append(dep.element) meta_element_queue.append(meta_dep) else: diff --git a/src/buildstream/exceptions.py b/src/buildstream/exceptions.py index cc2bd6381..2c455be84 100644 --- a/src/buildstream/exceptions.py +++ b/src/buildstream/exceptions.py @@ -139,3 +139,6 @@ class LoadErrorReason(Enum): DUPLICATE_DEPENDENCY = 24 """A duplicate dependency was detected""" + + LINK_FORBIDDEN_DEPENDENCIES = 25 + """A link element declared dependencies""" diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index 86d1de8f8..c9e78632f 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -48,13 +48,6 @@ Overview # Optionally look in a subpath of the source repository for the project path: projects/hello - # Optionally specify another junction element to serve as a target for - # this element. Target should be defined using the syntax - # ``{junction-name}:{element-name}``. - # - # Note that this option cannot be used in conjunction with sources. - target: sub-project.bst:sub-sub-project.bst - # Optionally declare whether elements within the junction project # should interact with project remotes (default: False). cache-junction-elements: False @@ -132,29 +125,24 @@ simply use one junction and ignore the others. Due to this, BuildStream requires the user to resolve possibly conflicting nested junctions by creating a junction with the same name in the top-level project, which then takes precedence. -Targeting other junctions -~~~~~~~~~~~~~~~~~~~~~~~~~ -When working with nested junctions, you can also create a junction element that -targets another junction element in the sub-project. This can be useful if you -need to ensure that both the top-level project and the sub-project are using -the same version of the sub-sub-project. +Linking to other junctions +~~~~~~~~~~~~~~~~~~~~~~~~~~ +When working with nested junctions, you often need to ensure that multiple +projects are using the same version of a given subproject. + +In order to ensure that your project is using a junction to a sub-subproject +declared by a direct subproject, then you can use a :mod:`link <elements.link>` +element in place of declaring a junction. -This can be done using the ``target`` configuration option. See below for an -example: +This lets you create a link to a junction in the subproject, which you +can then treat as a regular junction in your toplevel project. .. code:: yaml - kind: junction + kind: link config: target: subproject.bst:subsubproject.bst - -In the above example, this junction element would be targeting the junction -element named ``subsubproject.bst`` in the subproject referred to by -``subproject.bst``. - -Note that when targeting another junction, the names of the junction element -must not be the same as the name of the target. """ from buildstream import Element, ElementError @@ -173,39 +161,15 @@ class JunctionElement(Element): def configure(self, node): - node.validate_keys(["path", "options", "target", "cache-junction-elements", "ignore-junction-remotes"]) + node.validate_keys(["path", "options", "cache-junction-elements", "ignore-junction-remotes"]) self.path = node.get_str("path", default="") self.options = node.get_mapping("options", default={}) - self.target = node.get_str("target", default=None) - self.target_element = None - self.target_junction = None self.cache_junction_elements = node.get_bool("cache-junction-elements", default=False) self.ignore_junction_remotes = node.get_bool("ignore-junction-remotes", default=False) def preflight(self): - # "target" cannot be used in conjunction with: - # 1. sources - # 2. config['options'] - # 3. config['path'] - if self.target and any(self.sources()): - raise ElementError("junction elements cannot define both 'sources' and 'target' config option") - if self.target and any(self.options.items()): - raise ElementError("junction elements cannot define both 'options' and 'target'") - if self.target and self.path: - raise ElementError("junction elements cannot define both 'path' and 'target'") - - # Validate format of target, if defined - if self.target: - try: - self.target_junction, self.target_element = self.target.split(":") - except ValueError: - raise ElementError("'target' option must be in format '{junction-name}:{element-name}'") - - # We cannot target a junction that has the same name as us, since that - # will cause an infinite recursion while trying to load it. - if self.name == self.target_element: - raise ElementError("junction elements cannot target an element with the same name") + pass def get_unique_key(self): # Junctions do not produce artifacts. get_unique_key() implementation diff --git a/src/buildstream/plugins/elements/link.py b/src/buildstream/plugins/elements/link.py new file mode 100644 index 000000000..611108241 --- /dev/null +++ b/src/buildstream/plugins/elements/link.py @@ -0,0 +1,90 @@ +# +# Copyright (C) 2020 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +link - Link elements +================================ +This element is a link to another element, allowing one to create +a symbolic element which will be resolved to another element. + + +Overview +-------- +The only configuration allowed in a ``link`` element is the specified +target of the link. + +.. code:: yaml + + kind: link + + config: + target: element.bst + +The ``link`` element can be used to refer to elements in subprojects, and +can be used to symbolically link junctions as well as other elements. +""" + +from buildstream import Element + + +# Element implementation for the 'link' kind. +class LinkElement(Element): + # pylint: disable=attribute-defined-outside-init + + BST_MIN_VERSION = "2.0" + + # Links are not allowed any dependencies or sources + BST_FORBID_BDEPENDS = True + BST_FORBID_RDEPENDS = True + BST_FORBID_SOURCES = True + + def configure(self, node): + + node.validate_keys(["target"]) + + # Hold onto the provenance of the specified target, + # allowing the loader to raise errors with better context. + # + target_node = node.get_scalar("target") + self.target = target_node.as_str() + self.target_provenance = target_node.get_provenance() + + def preflight(self): + pass + + def get_unique_key(self): + # This is only used early on but later discarded + return 1 + + def configure_sandbox(self, sandbox): + assert False, "link elements should be discarded at load time" + + def stage(self, sandbox): + assert False, "link elements should be discarded at load time" + + def generate_script(self): + assert False, "link elements should be discarded at load time" + + def assemble(self, sandbox): + assert False, "link elements should be discarded at load time" + + +# Plugin entry point +def setup(): + return LinkElement |