diff options
Diffstat (limited to 'buildstream')
| -rw-r--r-- | buildstream/_sysroot_dependency_loader.py | 260 | ||||
| -rw-r--r-- | buildstream/buildelement.py | 26 | ||||
| -rw-r--r-- | buildstream/element.py | 8 | ||||
| -rw-r--r-- | buildstream/plugins/elements/script.py | 5 | ||||
| -rw-r--r-- | buildstream/scriptelement.py | 109 |
5 files changed, 300 insertions, 108 deletions
diff --git a/buildstream/_sysroot_dependency_loader.py b/buildstream/_sysroot_dependency_loader.py new file mode 100644 index 000000000..c9b348ceb --- /dev/null +++ b/buildstream/_sysroot_dependency_loader.py @@ -0,0 +1,260 @@ +# +# Copyright (C) 2019 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Valentin David <valentin.david@codethink.co.uk> + +import os +import itertools +from collections.abc import Mapping + +from .dependency_loader import DependencyLoader, Dependency +from . import Element, ElementError, Scope, SandboxFlags + + +# SysrootDependencyLoader(): +# +# `SysrootDependencyLoader` implements a `DependencyLoader` to extract +# sysroot'ed dependencies. +class SysrootDependencyLoader(DependencyLoader): + + def get_dependencies(self, node): + sysroots = self.node_get_member(node, list, 'sysroots', default=[]) + dependencies = [] + + for sysroot in sysroots: + depends = self.node_get_member(sysroot, list, 'depends', default=[]) + build_depends = self.node_get_member(sysroot, list, 'build-depends', default=[]) + depends_iter = itertools.product(['all'], depends) + build_depends_iter = itertools.product(['build'], build_depends) + for default_type, dep in itertools.chain(depends_iter, build_depends_iter): + if isinstance(dep, Mapping): + provenance = self.node_provenance(dep) + filename = self.node_get_member(node, str, 'filename') + dep_type = self.node_get_member(node, str, 'type', default=default_type) + junction = self.node_get_member(node, str, 'junction', default=None) + dependencies.append(Dependency(filename, dep_type=dep_type, + junction=junction, provenance=provenance)) + else: + provenance = self.node_provenance(sysroot) + dependencies.append(Dependency(dep, dep_type=default_type, provenance=provenance)) + + return dependencies + + +# SysrootHelper(): +# +# `SysrootHelper` should be used in element plugins that use +# `SysrootDependencyLoader` as dependency loader. It provides +# The implementation for staging. +class SysrootHelper: + + CONFIG_KEYS = ['sysroots'] + __layout = [] + + def __init__(self, element, node): + + self.__element = element + + for sysroot in self.__element.node_get_member(node, list, 'sysroots', []): + self.__element.node_validate(sysroot, ['path', 'depends', 'build-depends']) + path = self.__element.node_subst_member(sysroot, 'path') + depends = self.__element.node_get_member(sysroot, list, 'depends', default=[]) + build_depends = self.__element.node_get_member(sysroot, list, 'build-depends', default=[]) + for dep in itertools.chain(depends, build_depends): + if isinstance(dep, Mapping): + self.__element.node_validate(dep, ['filename', 'type', 'junction']) + filename = self.__element.node_get_member(dep, str, 'filename') + junction = self.__element.node_get_member(dep, str, 'junction', default=None) + else: + filename = dep + junction = None + self.layout_add(filename, path, junction=junction) + + # layout_add(): + # + # Adds a destination where a dependency should be staged + # + # Args: + # element (str): Element name of the dependency + # destination (str): Path where element will be staged + # junction (str): Junction of the dependency + # + # If `junction` is None, then the dependency should be in the same + # project as the current element. + # + # If `junction` is ignored or `Element.IGNORE_JUNCTION`, the + # junction of the dependency is not checked. This is for backward + # compliancy and should not be used. + # + # If `element` is None, the destination will just + # be marked in the sandbox. + def layout_add(self, element, destination, *, junction=Element.IGNORE_JUNCTION): + # + # Even if this is an empty list by default, make sure that its + # instance data instead of appending stuff directly onto class data. + # + if not self.__layout: + self.__layout = [] + item = {'element': element, + 'destination': destination} + if junction is not Element.IGNORE_JUNCTION: + item['junction'] = junction + self.__layout.append(item) + + # validate(): + # + # Verify that elements in layouts are dependencies. + # + # Raises: + # (ElementError): When a element is not in the dependencies + # + # This method is only useful when SysrootHelper.layout_add + # has been called directly. + # + # This should be called in implementation of Plugin.preflight. + def validate(self): + if self.__layout: + # Cannot proceed if layout specifies an element that isn't part + # of the dependencies. + for item in self.__layout: + if not item['element']: + if not self.__search(item): + raise ElementError("{}: '{}' in layout not found in dependencies" + .format(self.__element, item['element'])) + + # stage(): + # + # Stage dependencies and integrate root dependencies + # + # Args: + # stage_all (bool): Whether to stage all dependencies, not just the ones mapped + # + def stage(self, sandbox, stage_all): + + staged = set() + sysroots = {} + + for item in self.__layout: + + # Skip layout members which dont stage an element + if not item['element']: + continue + + element = self.__search(item) + staged.add(element) + if item['destination'] not in sysroots: + sysroots[item['destination']] = [element] + else: + sysroots[item['destination']].append(element) + + if stage_all or not self.__layout: + for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False): + if build_dep in staged: + continue + if '/' not in sysroots: + sysroots['/'] = [build_dep] + else: + sysroots['/'].append(build_dep) + + for sysroot, deps in sysroots.items(): + with self.__element.timed_activity("Staging dependencies at {}".format(sysroot), silent_nested=True): + if sysroot != '/': + virtual_dstdir = sandbox.get_virtual_directory() + virtual_dstdir.descend(sysroot.lstrip(os.sep).split(os.sep), create=True) + all_deps = set() + for dep in deps: + for run_dep in dep.dependencies(Scope.RUN): + all_deps.add(run_dep) + self.__element.stage_dependency_artifacts(sandbox, Scope.BUILD, path=sysroot, dependencies=all_deps) + + with sandbox.batch(SandboxFlags.NONE): + for item in self.__layout: + + # Skip layout members which dont stage an element + if not item['element']: + continue + + element = self.__search(item) + + # Integration commands can only be run for elements staged to / + if item['destination'] == '/': + with self.__element.timed_activity("Integrating {}".format(element.name), + silent_nested=True): + for dep in element.dependencies(Scope.RUN): + element.integrate(sandbox) + + if stage_all or not self.__layout: + for build_dep in self.__element.dependencies(Scope.BUILD, recurse=False): + if build_dep in staged: + continue + + with self.__element.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True): + for dep in build_dep.dependencies(Scope.RUN): + dep.integrate(sandbox) + + # has_sysroots(): + # + # Tells whether any element has been mapped + # + # Returns: + # (bool): Whether any element has been mapped + def has_sysroots(self): + return bool(self.__layout) + + # get_unique_key(): + # + # Returns a value usable for an element unique key + # + # Returns: + # (dict): A dictionary that uniquely identify the mapping configuration + def get_unique_key(self): + return self.__layout + + # configure_sandbox(): + # + # Configure the sandbox. Mark required directories in the sandbox. + # + # Args: + # extra_directories (list(str)): Extra directories to mark + # + # Because Sandbox.mark_directory should be called + # only once, marked directories should passed as `extra_directories` + # instead of being marked directly. + def configure_sandbox(self, sandbox, extra_directories): + + directories = {directory: False for directory in extra_directories} + + for item in self.__layout: + destination = item['destination'] + was_artifact = directories.get(destination, False) + directories[destination] = item['element'] or was_artifact + + for directory, artifact in directories.items(): + # Root does not need to be marked as it is always mounted + # with artifact (unless explicitly marked non-artifact) + if directory != '/': + sandbox.mark_directory(directory, artifact=artifact) + + # + # Private methods + # + + def __search(self, item): + if 'junction' in item: + return self.__element.search(Scope.BUILD, item['element'], junction=item['junction']) + else: + return self.__element.search(Scope.BUILD, item['element']) diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py index 6ef060f12..126c966b8 100644 --- a/buildstream/buildelement.py +++ b/buildstream/buildelement.py @@ -135,8 +135,10 @@ artifact collection purposes. """ import os -from . import Element, Scope + +from . import Element from . import SandboxFlags +from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper # This list is preserved because of an unfortunate situation, we @@ -157,17 +159,20 @@ _command_steps = ['configure-commands', class BuildElement(Element): + DEPENDENCY_LOADER = SysrootDependencyLoader + ############################################################# # Abstract Method Implementations # ############################################################# def configure(self, node): self.__commands = {} # pylint: disable=attribute-defined-outside-init + self.__sysroots = SysrootHelper(self, node) # pylint: disable=attribute-defined-outside-init # FIXME: Currently this forcefully validates configurations # for all BuildElement subclasses so they are unable to # extend the configuration - self.node_validate(node, _command_steps) + self.node_validate(node, _command_steps + SysrootHelper.CONFIG_KEYS) for command_name in _legacy_command_steps: if command_name in _command_steps: @@ -191,6 +196,9 @@ class BuildElement(Element): if self.get_variable('notparallel'): dictionary['notparallel'] = True + if self.__sysroots.has_sysroots(): + dictionary['sysroots'] = self.__sysroots.get_unique_key() + return dictionary def configure_sandbox(self, sandbox): @@ -198,8 +206,8 @@ class BuildElement(Element): install_root = self.get_variable('install-root') # Tell the sandbox to mount the build root and install root - sandbox.mark_directory(build_root) - sandbox.mark_directory(install_root) + self.__sysroots.configure_sandbox(sandbox, [build_root, + install_root]) # Allow running all commands in a specified subdirectory command_subdir = self.get_variable('command-subdir') @@ -217,15 +225,7 @@ class BuildElement(Element): def stage(self, sandbox): - # Stage deps in the sandbox root - with self.timed_activity("Staging dependencies", silent_nested=True): - self.stage_dependency_artifacts(sandbox, Scope.BUILD) - - # Run any integration commands provided by the dependencies - # once they are all staged and ready - with sandbox.batch(SandboxFlags.NONE, label="Integrating sandbox"): - for dep in self.dependencies(Scope.BUILD): - dep.integrate(sandbox) + self.__sysroots.stage(sandbox, True) # Stage sources in the build root self.stage_sources(sandbox, self.get_variable('build-root')) diff --git a/buildstream/element.py b/buildstream/element.py index fcf7c15eb..d28390f0d 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -677,7 +677,8 @@ class Element(Plugin): return link_result.combine(copy_result) def stage_dependency_artifacts(self, sandbox, scope, *, path=None, - include=None, exclude=None, orphans=True): + include=None, exclude=None, orphans=True, + dependencies=None): """Stage element dependencies in scope This is primarily a convenience wrapper around @@ -692,6 +693,7 @@ class Element(Plugin): include (list): An optional list of domains to include files from exclude (list): An optional list of domains to exclude files from orphans (bool): Whether to include files not spoken for by split domains + dependencies (list): An optional list of dependencies to stage Raises: (:class:`.ElementError`): If any of the dependencies in `scope` have not @@ -707,7 +709,9 @@ class Element(Plugin): if self.__can_build_incrementally() and workspace.last_successful: old_dep_keys = self.__get_artifact_metadata_dependencies(workspace.last_successful) - for dep in self.dependencies(scope): + if dependencies is None: + dependencies = self.dependencies(scope) + for dep in dependencies: # If we are workspaced, and we therefore perform an # incremental build, we must ensure that we update the mtimes # of any files created by our dependencies since the last diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py index 6c33ecf95..657584baa 100644 --- a/buildstream/plugins/elements/script.py +++ b/buildstream/plugins/elements/script.py @@ -36,6 +36,7 @@ The default configuration and possible options are as such: """ import buildstream +from buildstream._sysroot_dependency_loader import DependencyLoader # Element implementation for the 'script' kind. @@ -46,6 +47,8 @@ class ScriptElement(buildstream.ScriptElement): BST_VIRTUAL_DIRECTORY = True def configure(self, node): + super().configure(node) + for n in self.node_get_member(node, list, 'layout', []): dst = self.node_subst_member(n, 'destination') elm = self.node_subst_member(n, 'element', None) @@ -53,7 +56,7 @@ class ScriptElement(buildstream.ScriptElement): self.node_validate(node, [ 'commands', 'root-read-only', 'layout' - ]) + ] + self.COMMON_CONFIG_KEYS) cmds = self.node_subst_list(node, "commands") self.add_commands("commands", cmds) diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py index 697cd2822..703c5a938 100644 --- a/buildstream/scriptelement.py +++ b/buildstream/scriptelement.py @@ -35,7 +35,8 @@ implementations. import os from collections import OrderedDict -from . import Element, ElementError, Scope, SandboxFlags +from . import Element, SandboxFlags +from ._sysroot_dependency_loader import SysrootDependencyLoader, SysrootHelper class ScriptElement(Element): @@ -43,7 +44,6 @@ class ScriptElement(Element): __cwd = "/" __root_read_only = False __commands = None - __layout = [] # The compose element's output is its dependencies, so # we must rebuild if the dependencies change even when @@ -59,6 +59,15 @@ class ScriptElement(Element): # added, to reduce the potential for confusion BST_FORBID_SOURCES = True + COMMON_CONFIG_KEYS = SysrootHelper.CONFIG_KEYS + + DEPENDENCY_LOADER = SysrootDependencyLoader + + def configure(self, node): + + self.__stage_all = True # pylint: disable=attribute-defined-outside-init + self.__sysroots = SysrootHelper(self, node) # pylint: disable=attribute-defined-outside-init + def set_work_dir(self, work_dir=None): """Sets the working dir @@ -134,14 +143,8 @@ class ScriptElement(Element): In the case that no element is specified, a read-write directory will be made available at the specified location. """ - # - # Even if this is an empty list by default, make sure that its - # instance data instead of appending stuff directly onto class data. - # - if not self.__layout: - self.__layout = [] - self.__layout.append({"element": element, - "destination": destination}) + self.__stage_all = False # pylint: disable=attribute-defined-outside-init + self.__sysroots.layout_add(element, destination) def add_commands(self, group_name, command_list): """Adds a list of commands under the group-name. @@ -164,32 +167,15 @@ class ScriptElement(Element): self.__commands = OrderedDict() self.__commands[group_name] = command_list - def __validate_layout(self): - if self.__layout: - # Cannot proceeed if layout is used, but none are for "/" - root_defined = any([(entry['destination'] == '/') for entry in self.__layout]) - if not root_defined: - raise ElementError("{}: Using layout, but none are staged as '/'" - .format(self)) - - # Cannot proceed if layout specifies an element that isn't part - # of the dependencies. - for item in self.__layout: - if item['element']: - if not self.search(Scope.BUILD, item['element']): - raise ElementError("{}: '{}' in layout not found in dependencies" - .format(self, item['element'])) - def preflight(self): - # The layout, if set, must make sense. - self.__validate_layout() + self.__sysroots.validate() def get_unique_key(self): return { 'commands': self.__commands, 'cwd': self.__cwd, 'install-root': self.__install_root, - 'layout': self.__layout, + 'layout': self.__sysroots.get_unique_key(), 'root-read-only': self.__root_read_only } @@ -201,72 +187,11 @@ class ScriptElement(Element): # Setup environment sandbox.set_environment(self.get_environment()) - # Tell the sandbox to mount the install root - directories = {self.__install_root: False} - - # Mark the artifact directories in the layout - for item in self.__layout: - destination = item['destination'] - was_artifact = directories.get(destination, False) - directories[destination] = item['element'] or was_artifact - - for directory, artifact in directories.items(): - # Root does not need to be marked as it is always mounted - # with artifact (unless explicitly marked non-artifact) - if directory != '/': - sandbox.mark_directory(directory, artifact=artifact) + self.__sysroots.configure_sandbox(sandbox, [self.__install_root]) def stage(self, sandbox): - # Stage the elements, and run integration commands where appropriate. - if not self.__layout: - # if no layout set, stage all dependencies into / - for build_dep in self.dependencies(Scope.BUILD, recurse=False): - with self.timed_activity("Staging {} at /" - .format(build_dep.name), silent_nested=True): - build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/") - - with sandbox.batch(SandboxFlags.NONE): - for build_dep in self.dependencies(Scope.BUILD, recurse=False): - with self.timed_activity("Integrating {}".format(build_dep.name), silent_nested=True): - for dep in build_dep.dependencies(Scope.RUN): - dep.integrate(sandbox) - else: - # If layout, follow its rules. - for item in self.__layout: - - # Skip layout members which dont stage an element - if not item['element']: - continue - - element = self.search(Scope.BUILD, item['element']) - if item['destination'] == '/': - with self.timed_activity("Staging {} at /".format(element.name), - silent_nested=True): - element.stage_dependency_artifacts(sandbox, Scope.RUN) - else: - with self.timed_activity("Staging {} at {}" - .format(element.name, item['destination']), - silent_nested=True): - virtual_dstdir = sandbox.get_virtual_directory() - virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True) - element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination']) - - with sandbox.batch(SandboxFlags.NONE): - for item in self.__layout: - - # Skip layout members which dont stage an element - if not item['element']: - continue - - element = self.search(Scope.BUILD, item['element']) - - # Integration commands can only be run for elements staged to / - if item['destination'] == '/': - with self.timed_activity("Integrating {}".format(element.name), - silent_nested=True): - for dep in element.dependencies(Scope.RUN): - dep.integrate(sandbox) + self.__sysroots.stage(sandbox, self.__stage_all) install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep) sandbox.get_virtual_directory().descend(install_root_path_components, create=True) |