diff options
Diffstat (limited to 'src/buildstream/plugins/elements')
31 files changed, 2202 insertions, 0 deletions
diff --git a/src/buildstream/plugins/elements/__init__.py b/src/buildstream/plugins/elements/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/buildstream/plugins/elements/__init__.py diff --git a/src/buildstream/plugins/elements/autotools.py b/src/buildstream/plugins/elements/autotools.py new file mode 100644 index 000000000..7a05336b7 --- /dev/null +++ b/src/buildstream/plugins/elements/autotools.py @@ -0,0 +1,75 @@ +# +# Copyright (C) 2016, 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +autotools - Autotools build element +=================================== +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using Autotools build scripts (also known as the `GNU Build System +<https://en.wikipedia.org/wiki/GNU_Build_System>`_). + +You will often want to pass additional arguments to ``configure``. This should +be done on a per-element basis by setting the ``conf-local`` variable. Here is +an example: + +.. code:: yaml + + variables: + conf-local: | + --disable-foo --enable-bar + +If you want to pass extra options to ``configure`` for every element in your +project, set the ``conf-global`` variable in your project.conf file. Here is +an example of that: + +.. code:: yaml + + elements: + autotools: + variables: + conf-global: | + --disable-gtk-doc --disable-static + +Here is the default configuration for the ``autotools`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/autotools.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'autotools' kind. +class AutotoolsElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return AutotoolsElement diff --git a/src/buildstream/plugins/elements/autotools.yaml b/src/buildstream/plugins/elements/autotools.yaml new file mode 100644 index 000000000..85f7393e7 --- /dev/null +++ b/src/buildstream/plugins/elements/autotools.yaml @@ -0,0 +1,129 @@ +# Autotools default configurations + +variables: + + autogen: | + export NOCONFIGURE=1; + + if [ -x %{conf-cmd} ]; then true; + elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen; + elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh; + elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap; + elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh; + else autoreconf -ivf %{conf-root}; + fi + + # Project-wide extra arguments to be passed to `configure` + conf-global: '' + + # Element-specific extra arguments to be passed to `configure`. + conf-local: '' + + # For backwards compatibility only, do not use. + conf-extra: '' + + conf-cmd: "%{conf-root}/configure" + + conf-args: | + + --prefix=%{prefix} \ + --exec-prefix=%{exec_prefix} \ + --bindir=%{bindir} \ + --sbindir=%{sbindir} \ + --sysconfdir=%{sysconfdir} \ + --datadir=%{datadir} \ + --includedir=%{includedir} \ + --libdir=%{libdir} \ + --libexecdir=%{libexecdir} \ + --localstatedir=%{localstatedir} \ + --sharedstatedir=%{sharedstatedir} \ + --mandir=%{mandir} \ + --infodir=%{infodir} %{conf-extra} %{conf-global} %{conf-local} + + configure: | + + %{conf-cmd} %{conf-args} + + make: make + make-install: make -j1 DESTDIR="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + + + # Automatically remove libtool archive files + # + # Set remove-libtool-modules to "true" to remove .la files for + # modules intended to be opened with lt_dlopen() + # + # Set remove-libtool-libraries to "true" to remove .la files for + # libraries + # + # Value must be "true" or "false" + remove-libtool-modules: "false" + remove-libtool-libraries: "false" + + delete-libtool-archives: | + if %{remove-libtool-modules} || %{remove-libtool-libraries}; then + find "%{install-root}" -name "*.la" -print0 | while read -d '' -r file; do + if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then + if %{remove-libtool-modules}; then + echo "Removing ${file}." + rm "${file}" + else + echo "Not removing ${file}." + fi + else + if %{remove-libtool-libraries}; then + echo "Removing ${file}." + rm "${file}" + else + echo "Not removing ${file}." + fi + fi + done + fi + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{autogen} + - | + %{configure} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + - | + %{delete-libtool-archives} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# affect build output. +environment-nocache: +- MAKEFLAGS +- V diff --git a/src/buildstream/plugins/elements/cmake.py b/src/buildstream/plugins/elements/cmake.py new file mode 100644 index 000000000..74da04899 --- /dev/null +++ b/src/buildstream/plugins/elements/cmake.py @@ -0,0 +1,74 @@ +# +# Copyright (C) 2016, 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +cmake - CMake build element +=========================== +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using the `CMake <https://cmake.org/>`_ build system. + +You will often want to pass additional arguments to the ``cmake`` program for +specific configuration options. This should be done on a per-element basis by +setting the ``cmake-local`` variable. Here is an example: + +.. code:: yaml + + variables: + cmake-local: | + -DCMAKE_BUILD_TYPE=Debug + +If you want to pass extra options to ``cmake`` for every element in your +project, set the ``cmake-global`` variable in your project.conf file. Here is +an example of that: + +.. code:: yaml + + elements: + cmake: + variables: + cmake-global: | + -DCMAKE_BUILD_TYPE=Release + +Here is the default configuration for the ``cmake`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/cmake.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'cmake' kind. +class CMakeElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return CMakeElement diff --git a/src/buildstream/plugins/elements/cmake.yaml b/src/buildstream/plugins/elements/cmake.yaml new file mode 100644 index 000000000..ba20d7ce6 --- /dev/null +++ b/src/buildstream/plugins/elements/cmake.yaml @@ -0,0 +1,72 @@ +# CMake default configuration + +variables: + + build-dir: _builddir + + # Project-wide extra arguments to be passed to `cmake` + cmake-global: '' + + # Element-specific extra arguments to be passed to `cmake`. + cmake-local: '' + + # For backwards compatibility only, do not use. + cmake-extra: '' + + # The cmake generator to use + generator: Unix Makefiles + + cmake-args: | + + -DCMAKE_INSTALL_PREFIX:PATH="%{prefix}" \ + -DCMAKE_INSTALL_LIBDIR:PATH="%{lib}" %{cmake-extra} %{cmake-global} %{cmake-local} + + cmake: | + + cmake -B%{build-dir} -H"%{conf-root}" -G"%{generator}" %{cmake-args} + + make: cmake --build %{build-dir} -- ${JOBS} + make-install: env DESTDIR="%{install-root}" cmake --build %{build-dir} --target install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{cmake} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + JOBS: -j%{max-jobs} + V: 1 + +# And dont consider JOBS or V as something which may +# affect build output. +environment-nocache: +- JOBS +- V diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py new file mode 100644 index 000000000..b672cde0c --- /dev/null +++ b/src/buildstream/plugins/elements/compose.py @@ -0,0 +1,194 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +compose - Compose the output of multiple elements +================================================= +This element creates a selective composition of its dependencies. + +This is normally used at near the end of a pipeline to prepare +something for later deployment. + +Since this element's output includes its dependencies, it may only +depend on elements as `build` type dependencies. + +The default configuration and possible options are as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/compose.yaml + :language: yaml +""" + +import os +from buildstream import Element, Scope + + +# Element implementation for the 'compose' kind. +class ComposeElement(Element): + # pylint: disable=attribute-defined-outside-init + + # The compose element's output is its dependencies, so + # we must rebuild if the dependencies change even when + # not in strict build plans. + # + BST_STRICT_REBUILD = True + + # Compose artifacts must never have indirect dependencies, + # so runtime dependencies are forbidden. + BST_FORBID_RDEPENDS = True + + # This element ignores sources, so we should forbid them from being + # added, to reduce the potential for confusion + BST_FORBID_SOURCES = True + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + def configure(self, node): + self.node_validate(node, [ + 'integrate', 'include', 'exclude', 'include-orphans' + ]) + + # We name this variable 'integration' only to avoid + # collision with the Element.integrate() method. + self.integration = self.node_get_member(node, bool, 'integrate') + self.include = self.node_get_member(node, list, 'include') + self.exclude = self.node_get_member(node, list, 'exclude') + self.include_orphans = self.node_get_member(node, bool, 'include-orphans') + + def preflight(self): + pass + + def get_unique_key(self): + key = {'integrate': self.integration, + 'include': sorted(self.include), + 'orphans': self.include_orphans} + + if self.exclude: + key['exclude'] = sorted(self.exclude) + + return key + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + + require_split = self.include or self.exclude or not self.include_orphans + + # Stage deps in the sandbox root + with self.timed_activity("Staging dependencies", silent_nested=True): + self.stage_dependency_artifacts(sandbox, Scope.BUILD) + + manifest = set() + if require_split: + with self.timed_activity("Computing split", silent_nested=True): + for dep in self.dependencies(Scope.BUILD): + files = dep.compute_manifest(include=self.include, + exclude=self.exclude, + orphans=self.include_orphans) + manifest.update(files) + + # Make a snapshot of all the files. + vbasedir = sandbox.get_virtual_directory() + modified_files = set() + removed_files = set() + added_files = set() + + # Run any integration commands provided by the dependencies + # once they are all staged and ready + if self.integration: + with self.timed_activity("Integrating sandbox"): + if require_split: + + # Make a snapshot of all the files before integration-commands are run. + snapshot = set(vbasedir.list_relative_paths()) + vbasedir.mark_unmodified() + + with sandbox.batch(0): + for dep in self.dependencies(Scope.BUILD): + dep.integrate(sandbox) + + if require_split: + # Calculate added, modified and removed files + post_integration_snapshot = vbasedir.list_relative_paths() + modified_files = set(vbasedir.list_modified_paths()) + basedir_contents = set(post_integration_snapshot) + for path in manifest: + if path in snapshot and path not in basedir_contents: + removed_files.add(path) + + for path in basedir_contents: + if path not in snapshot: + added_files.add(path) + self.info("Integration modified {}, added {} and removed {} files" + .format(len(modified_files), len(added_files), len(removed_files))) + + # The remainder of this is expensive, make an early exit if + # we're not being selective about what is to be included. + if not require_split: + return '/' + + # Do we want to force include files which were modified by + # the integration commands, even if they were not added ? + # + manifest.update(added_files) + manifest.difference_update(removed_files) + + # XXX We should be moving things outside of the build sandbox + # instead of into a subdir. The element assemble() method should + # support this in some way. + # + installdir = vbasedir.descend('buildstream', 'install', create=True) + + # We already saved the manifest for created files in the integration phase, + # now collect the rest of the manifest. + # + + lines = [] + if self.include: + lines.append("Including files from domains: " + ", ".join(self.include)) + else: + lines.append("Including files from all domains") + + if self.exclude: + lines.append("Excluding files from domains: " + ", ".join(self.exclude)) + + if self.include_orphans: + lines.append("Including orphaned files") + else: + lines.append("Excluding orphaned files") + + detail = "\n".join(lines) + + def import_filter(path): + return path in manifest + + with self.timed_activity("Creating composition", detail=detail, silent_nested=True): + self.info("Composing {} files".format(len(manifest))) + installdir.import_files(vbasedir, filter_callback=import_filter, can_link=True) + + # And we're done + return os.path.join(os.sep, 'buildstream', 'install') + + +# Plugin entry point +def setup(): + return ComposeElement diff --git a/src/buildstream/plugins/elements/compose.yaml b/src/buildstream/plugins/elements/compose.yaml new file mode 100644 index 000000000..fd2eb9358 --- /dev/null +++ b/src/buildstream/plugins/elements/compose.yaml @@ -0,0 +1,34 @@ + +# Compose element configuration +config: + + # Whether to run the integration commands for the + # staged dependencies. + # + integrate: True + + # A list of domains to include from each artifact, as + # they were defined in the element's 'split-rules'. + # + # Since domains can be added, it is not an error to + # specify domains which may not exist for all of the + # elements in this composition. + # + # The default empty list indicates that all domains + # from each dependency should be included. + # + include: [] + + # A list of domains to exclude from each artifact, as + # they were defined in the element's 'split-rules'. + # + # In the case that a file is spoken for by a domain + # in the 'include' list and another in the 'exclude' + # list, then the file will be excluded. + exclude: [] + + # Whether to include orphan files which are not + # included by any of the 'split-rules' present on + # a given element. + # + include-orphans: True diff --git a/src/buildstream/plugins/elements/distutils.py b/src/buildstream/plugins/elements/distutils.py new file mode 100644 index 000000000..4b2c1e2f4 --- /dev/null +++ b/src/buildstream/plugins/elements/distutils.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +distutils - Python distutils element +==================================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +python distutils + +The distutils default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/distutils.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the python 'distutils' kind. +class DistutilsElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return DistutilsElement diff --git a/src/buildstream/plugins/elements/distutils.yaml b/src/buildstream/plugins/elements/distutils.yaml new file mode 100644 index 000000000..cec7da6e9 --- /dev/null +++ b/src/buildstream/plugins/elements/distutils.yaml @@ -0,0 +1,49 @@ +# Default python distutils configuration + +variables: + + # When building for python2 distutils, simply + # override this in the element declaration + python: python3 + + python-build: | + + %{python} %{conf-root}/setup.py build + + install-args: | + + --prefix "%{prefix}" \ + --root "%{install-root}" + + python-install: | + + %{python} %{conf-root}/setup.py install %{install-args} + + +config: + + # Commands for configuring the software + # + configure-commands: [] + + # Commands for building the software + # + build-commands: + - | + %{python-build} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{python-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + - | + %{fix-pyc-timestamps} diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py new file mode 100644 index 000000000..45847e685 --- /dev/null +++ b/src/buildstream/plugins/elements/filter.py @@ -0,0 +1,256 @@ +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jonathan Maw <jonathan.maw@codethink.co.uk> + +""" +filter - Extract a subset of files from another element +======================================================= +Filter another element by producing an output that is a subset of +the parent element's output. Subsets are defined by the parent element's +:ref:`split rules <public_split_rules>`. + +Overview +-------- +A filter element must have exactly one *build* dependency, where said +dependency is the 'parent' element which we would like to filter. +Runtime dependencies may also be specified, which can be useful to propagate +forward from this filter element onto its reverse dependencies. +See :ref:`Dependencies <format_dependencies>` to see how we specify dependencies. + +When workspaces are opened, closed or reset on a filter element, or this +element is tracked, the filter element will transparently pass on the command +to its parent element (the sole build-dependency). + +Example +------- +Consider a simple import element, ``import.bst`` which imports the local files +'foo', 'bar' and 'baz' (each stored in ``files/``, relative to the project's root): + +.. code:: yaml + + kind: import + + # Specify sources to import + sources: + - kind: local + path: files + + # Specify public domain data, visible to other elements + public: + bst: + split-rules: + foo: + - /foo + bar: + - /bar + +.. note:: + + We can make an element's metadata visible to all reverse dependencies by making use + of the ``public:`` field. See the :ref:`public data documentation <format_public>` + for more information. + +In this example, ``import.bst`` will serve as the 'parent' of the filter element, thus +its output will be filtered. It is important to understand that the artifact of the +above element will contain the files: 'foo', 'bar' and 'baz'. + +Now, to produce an element whose artifact contains the file 'foo', and exlusively 'foo', +we can define the following filter, ``filter-foo.bst``: + +.. code:: yaml + + kind: filter + + # Declare the sole build-dependency of the filter element + depends: + - filename: import.bst + type: build + + # Declare a list of domains to include in the filter's artifact + config: + include: + - foo + +.. note:: + + We can also specify build-dependencies with a 'build-depends' field which has been + available since :ref:`format version 14 <project_format_version>`. See the + :ref:`Build-Depends documentation <format_build_depends>` for more detail. + +It should be noted that an 'empty' ``include:`` list would, by default, include all +split-rules specified in the parent element, which, in this example, would be the +files 'foo' and 'bar' (the file 'baz' was not covered by any split rules). + +Equally, we can use the ``exclude:`` statement to create the same artifact (which +only contains the file 'foo') by declaring the following element, ``exclude-bar.bst``: + +.. code:: yaml + + kind: filter + + # Declare the sole build-dependency of the filter element + depends: + - filename: import.bst + type: build + + # Declare a list of domains to exclude in the filter's artifact + config: + exclude: + - bar + +In addition to the ``include:`` and ``exclude:`` fields, there exists an ``include-orphans:`` +(Boolean) field, which defaults to ``False``. This will determine whether to include files +which are not present in the 'split-rules'. For example, if we wanted to filter out all files +which are not included as split rules we can define the following element, ``filter-misc.bst``: + +.. code:: yaml + + kind: filter + + # Declare the sole build-dependency of the filter element + depends: + - filename: import.bst + type: build + + # Filter out all files which are not declared as split rules + config: + exclude: + - foo + - bar + include-orphans: True + +The artifact of ``filter-misc.bst`` will only contain the file 'baz'. + +Below is more information regarding the the default configurations and possible options +of the filter element: + +.. literalinclude:: ../../../src/buildstream/plugins/elements/filter.yaml + :language: yaml +""" + +from buildstream import Element, ElementError, Scope + + +class FilterElement(Element): + # pylint: disable=attribute-defined-outside-init + + BST_ARTIFACT_VERSION = 1 + + # The filter element's output is its dependencies, so + # we must rebuild if the dependencies change even when + # not in strict build plans. + BST_STRICT_REBUILD = True + + # This element ignores sources, so we should forbid them from being + # added, to reduce the potential for confusion + BST_FORBID_SOURCES = True + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + # Filter elements do not run any commands + BST_RUN_COMMANDS = False + + def configure(self, node): + self.node_validate(node, [ + 'include', 'exclude', 'include-orphans' + ]) + + self.include = self.node_get_member(node, list, 'include') + self.exclude = self.node_get_member(node, list, 'exclude') + self.include_orphans = self.node_get_member(node, bool, 'include-orphans') + self.include_provenance = self.node_provenance(node, member_name='include') + self.exclude_provenance = self.node_provenance(node, member_name='exclude') + + def preflight(self): + # Exactly one build-depend is permitted + build_deps = list(self.dependencies(Scope.BUILD, recurse=False)) + if len(build_deps) != 1: + detail = "Full list of build-depends:\n" + deps_list = " \n".join([x.name for x in build_deps]) + detail += deps_list + raise ElementError("{}: {} element must have exactly 1 build-dependency, actually have {}" + .format(self, type(self).__name__, len(build_deps)), + detail=detail, reason="filter-bdepend-wrong-count") + + # That build-depend must not also be a runtime-depend + runtime_deps = list(self.dependencies(Scope.RUN, recurse=False)) + if build_deps[0] in runtime_deps: + detail = "Full list of runtime depends:\n" + deps_list = " \n".join([x.name for x in runtime_deps]) + detail += deps_list + raise ElementError("{}: {} element's build dependency must not also be a runtime dependency" + .format(self, type(self).__name__), + detail=detail, reason="filter-bdepend-also-rdepend") + + def get_unique_key(self): + key = { + 'include': sorted(self.include), + 'exclude': sorted(self.exclude), + 'orphans': self.include_orphans, + } + return key + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + with self.timed_activity("Staging artifact", silent_nested=True): + for dep in self.dependencies(Scope.BUILD, recurse=False): + # Check that all the included/excluded domains exist + pub_data = dep.get_public_data('bst') + split_rules = self.node_get_member(pub_data, dict, 'split-rules', {}) + unfound_includes = [] + for domain in self.include: + if domain not in split_rules: + unfound_includes.append(domain) + unfound_excludes = [] + for domain in self.exclude: + if domain not in split_rules: + unfound_excludes.append(domain) + + detail = [] + if unfound_includes: + detail.append("Unknown domains were used in {}".format(self.include_provenance)) + detail.extend([' - {}'.format(domain) for domain in unfound_includes]) + + if unfound_excludes: + detail.append("Unknown domains were used in {}".format(self.exclude_provenance)) + detail.extend([' - {}'.format(domain) for domain in unfound_excludes]) + + if detail: + detail = '\n'.join(detail) + raise ElementError("Unknown domains declared.", detail=detail) + + dep.stage_artifact(sandbox, include=self.include, + exclude=self.exclude, orphans=self.include_orphans) + return "" + + def _get_source_element(self): + # Filter elements act as proxies for their sole build-dependency + build_deps = list(self.dependencies(Scope.BUILD, recurse=False)) + assert len(build_deps) == 1 + output_elm = build_deps[0]._get_source_element() + return output_elm + + +def setup(): + return FilterElement diff --git a/src/buildstream/plugins/elements/filter.yaml b/src/buildstream/plugins/elements/filter.yaml new file mode 100644 index 000000000..9c2bf69f4 --- /dev/null +++ b/src/buildstream/plugins/elements/filter.yaml @@ -0,0 +1,29 @@ + +# Filter element configuration +config: + + # A list of domains to include in each artifact, as + # they were defined as public data in the parent + # element's 'split-rules'. + # + # If a domain is specified that does not exist, the + # filter element will fail to build. + # + # The default empty list indicates that all domains + # of the parent's artifact should be included. + # + include: [] + + # A list of domains to exclude from each artifact, as + # they were defined in the parent element's 'split-rules'. + # + # In the case that a file is spoken for by a domain + # in the 'include' list and another in the 'exclude' + # list, then the file will be excluded. + exclude: [] + + # Whether to include orphan files which are not + # included by any of the 'split-rules' present in + # the parent element. + # + include-orphans: False diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py new file mode 100644 index 000000000..61e353dbc --- /dev/null +++ b/src/buildstream/plugins/elements/import.py @@ -0,0 +1,129 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +import - Import sources directly +================================ +Import elements produce artifacts directly from its sources +without any kind of processing. These are typically used to +import an SDK to build on top of or to overlay your build with +some configuration data. + +The empty configuration is as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/import.yaml + :language: yaml +""" + +import os +from buildstream import Element, ElementError + + +# Element implementation for the 'import' kind. +class ImportElement(Element): + # pylint: disable=attribute-defined-outside-init + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + # Import elements do not run any commands + BST_RUN_COMMANDS = False + + def configure(self, node): + self.node_validate(node, [ + 'source', 'target' + ]) + + self.source = self.node_subst_member(node, 'source') + self.target = self.node_subst_member(node, 'target') + + def preflight(self): + # Assert that we have at least one source to fetch. + + sources = list(self.sources()) + if not sources: + raise ElementError("{}: An import element must have at least one source.".format(self)) + + def get_unique_key(self): + return { + 'source': self.source, + 'target': self.target + } + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + + # Stage sources into the input directory + # Do not mount workspaces as the files are copied from outside the sandbox + self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False) + + rootdir = sandbox.get_virtual_directory() + inputdir = rootdir.descend('input') + outputdir = rootdir.descend('output', create=True) + + # The directory to grab + inputdir = inputdir.descend(*self.source.strip(os.sep).split(os.sep)) + + # The output target directory + outputdir = outputdir.descend(*self.target.strip(os.sep).split(os.sep), create=True) + + if inputdir.is_empty(): + raise ElementError("{}: No files were found inside directory '{}'" + .format(self, self.source)) + + # Move it over + outputdir.import_files(inputdir) + + # And we're done + return '/output' + + def generate_script(self): + build_root = self.get_variable('build-root') + install_root = self.get_variable('install-root') + commands = [] + + # The directory to grab + inputdir = os.path.join(build_root, self.normal_name, self.source.lstrip(os.sep)) + inputdir = inputdir.rstrip(os.sep) + + # The output target directory + outputdir = os.path.join(install_root, self.target.lstrip(os.sep)) + outputdir = outputdir.rstrip(os.sep) + + # Ensure target directory parent exists but target directory doesn't + commands.append("mkdir -p {}".format(os.path.dirname(outputdir))) + commands.append("[ ! -e {outputdir} ] || rmdir {outputdir}".format(outputdir=outputdir)) + + # Move it over + commands.append("mv {} {}".format(inputdir, outputdir)) + + script = "" + for cmd in commands: + script += "(set -ex; {}\n) || exit 1\n".format(cmd) + + return script + + +# Plugin entry point +def setup(): + return ImportElement diff --git a/src/buildstream/plugins/elements/import.yaml b/src/buildstream/plugins/elements/import.yaml new file mode 100644 index 000000000..698111b55 --- /dev/null +++ b/src/buildstream/plugins/elements/import.yaml @@ -0,0 +1,14 @@ +# The import element simply stages the given sources +# directly to the root of the sandbox and then collects +# the output to create an output artifact. +# +config: + + # By default we collect everything staged, specify a + # directory here to output only a subset of the staged + # input sources. + source: / + + # Prefix the output with an optional directory, by default + # the input is found at the root of the produced artifact. + target: / diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py new file mode 100644 index 000000000..15ef115d9 --- /dev/null +++ b/src/buildstream/plugins/elements/junction.py @@ -0,0 +1,229 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jürg Billeter <juerg.billeter@codethink.co.uk> + +""" +junction - Integrate subprojects +================================ +This element is a link to another BuildStream project. It allows integration +of multiple projects into a single pipeline. + +Overview +-------- + +.. code:: yaml + + kind: junction + + # Specify the BuildStream project source + sources: + - kind: git + url: upstream:projectname.git + track: master + ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6 + + # Specify the junction configuration + config: + + # Override project options + options: + machine_arch: "%{machine_arch}" + debug: True + + # Optionally look in a subpath of the source repository for the project + path: projects/hello + + # Optionally specify another junction element to serve as a target for + # this element. Target should be defined using the syntax + # ``{junction-name}:{element-name}``. + # + # Note that this option cannot be used in conjunction with sources. + target: sub-project.bst:sub-sub-project.bst + +.. note:: + + The configuration option to allow specifying junction targets is available + since :ref:`format version 24 <project_format_version>`. + +.. note:: + + Junction elements may not specify any dependencies as they are simply + links to other projects and are not in the dependency graph on their own. + +With a junction element in place, local elements can depend on elements in +the other BuildStream project using the additional ``junction`` attribute in the +dependency dictionary: + +.. code:: yaml + + depends: + - junction: toolchain.bst + filename: gcc.bst + type: build + +While junctions are elements, only a limited set of element operations is +supported. They can be tracked and fetched like other elements. +However, junction elements do not produce any artifacts, which means that +they cannot be built or staged. It also means that another element cannot +depend on a junction element itself. + +.. note:: + + BuildStream does not implicitly track junction elements. This means + that if we were to invoke: `bst build --track-all ELEMENT` on an element + which uses a junction element, the ref of the junction element + will not automatically be updated if a more recent version exists. + + Therefore, if you require the most up-to-date version of a subproject, + you must explicitly track the junction element by invoking: + `bst source track JUNCTION_ELEMENT`. + + Furthermore, elements within the subproject are also not tracked by default. + For this, we must specify the `--track-cross-junctions` option. This option + must be preceeded by `--track ELEMENT` or `--track-all`. + + +Sources +------- +``bst show`` does not implicitly fetch junction sources if they haven't been +cached yet. However, they can be fetched explicitly: + +.. code:: + + bst source fetch junction.bst + +Other commands such as ``bst build`` implicitly fetch junction sources. + +Options +------- +.. code:: yaml + + options: + machine_arch: "%{machine_arch}" + debug: True + +Junctions can configure options of the linked project. Options are never +implicitly inherited across junctions, however, variables can be used to +explicitly assign the same value to a subproject option. + +.. _core_junction_nested: + +Nested Junctions +---------------- +Junctions can be nested. That is, subprojects are allowed to have junctions on +their own. Nested junctions in different subprojects may point to the same +project, however, in most use cases the same project should be loaded only once. +BuildStream uses the junction element name as key to determine which junctions +to merge. It is recommended that the name of a junction is set to the same as +the name of the linked project. + +As the junctions may differ in source version and options, BuildStream cannot +simply use one junction and ignore the others. Due to this, BuildStream requires +the user to resolve possibly conflicting nested junctions by creating a junction +with the same name in the top-level project, which then takes precedence. + +Targeting other junctions +~~~~~~~~~~~~~~~~~~~~~~~~~ +When working with nested junctions, you can also create a junction element that +targets another junction element in the sub-project. This can be useful if you +need to ensure that both the top-level project and the sub-project are using +the same version of the sub-sub-project. + +This can be done using the ``target`` configuration option. See below for an +example: + +.. code:: yaml + + kind: junction + + config: + target: subproject.bst:subsubproject.bst + +In the above example, this junction element would be targeting the junction +element named ``subsubproject.bst`` in the subproject referred to by +``subproject.bst``. + +Note that when targeting another junction, the names of the junction element +must not be the same as the name of the target. +""" + +from collections.abc import Mapping +from buildstream import Element, ElementError +from buildstream._pipeline import PipelineError + + +# Element implementation for the 'junction' kind. +class JunctionElement(Element): + # pylint: disable=attribute-defined-outside-init + + # Junctions are not allowed any dependencies + BST_FORBID_BDEPENDS = True + BST_FORBID_RDEPENDS = True + + def configure(self, node): + self.path = self.node_get_member(node, str, 'path', default='') + self.options = self.node_get_member(node, Mapping, 'options', default={}) + self.target = self.node_get_member(node, str, 'target', default=None) + self.target_element = None + self.target_junction = None + + def preflight(self): + # "target" cannot be used in conjunction with: + # 1. sources + # 2. config['options'] + # 3. config['path'] + if self.target and any(self.sources()): + raise ElementError("junction elements cannot define both 'sources' and 'target' config option") + if self.target and any(self.node_items(self.options)): + raise ElementError("junction elements cannot define both 'options' and 'target'") + if self.target and self.path: + raise ElementError("junction elements cannot define both 'path' and 'target'") + + # Validate format of target, if defined + if self.target: + try: + self.target_junction, self.target_element = self.target.split(":") + except ValueError: + raise ElementError("'target' option must be in format '{junction-name}:{element-name}'") + + # We cannot target a junction that has the same name as us, since that + # will cause an infinite recursion while trying to load it. + if self.name == self.target_element: + raise ElementError("junction elements cannot target an element with the same name") + + def get_unique_key(self): + # Junctions do not produce artifacts. get_unique_key() implementation + # is still required for `bst source fetch`. + return 1 + + def configure_sandbox(self, sandbox): + raise PipelineError("Cannot build junction elements") + + def stage(self, sandbox): + raise PipelineError("Cannot stage junction elements") + + def generate_script(self): + raise PipelineError("Cannot build junction elements") + + def assemble(self, sandbox): + raise PipelineError("Cannot build junction elements") + + +# Plugin entry point +def setup(): + return JunctionElement diff --git a/src/buildstream/plugins/elements/make.py b/src/buildstream/plugins/elements/make.py new file mode 100644 index 000000000..67a261100 --- /dev/null +++ b/src/buildstream/plugins/elements/make.py @@ -0,0 +1,56 @@ +# +# Copyright Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Ed Baunton <ebaunton1@bloomberg.net> + +""" +make - Make build element +========================= +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using GNU make based build. + +.. note:: + + The ``make`` element is available since :ref:`format version 9 <project_format_version>` + +Here is the default configuration for the ``make`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/make.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'make' kind. +class MakeElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return MakeElement diff --git a/src/buildstream/plugins/elements/make.yaml b/src/buildstream/plugins/elements/make.yaml new file mode 100644 index 000000000..83e5c658f --- /dev/null +++ b/src/buildstream/plugins/elements/make.yaml @@ -0,0 +1,42 @@ +# make default configurations + +variables: + make: make PREFIX="%{prefix}" + make-install: make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# affect build output. +environment-nocache: +- MAKEFLAGS +- V diff --git a/src/buildstream/plugins/elements/makemaker.py b/src/buildstream/plugins/elements/makemaker.py new file mode 100644 index 000000000..7da051592 --- /dev/null +++ b/src/buildstream/plugins/elements/makemaker.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +makemaker - Perl MakeMaker build element +======================================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +the Perl ExtUtil::MakeMaker build system + +The MakeMaker default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/makemaker.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'makemaker' kind. +class MakeMakerElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return MakeMakerElement diff --git a/src/buildstream/plugins/elements/makemaker.yaml b/src/buildstream/plugins/elements/makemaker.yaml new file mode 100644 index 000000000..c9c4622cb --- /dev/null +++ b/src/buildstream/plugins/elements/makemaker.yaml @@ -0,0 +1,48 @@ +# Default configuration for the Perl ExtUtil::MakeMaker +# build system + +variables: + + # To install perl distributions into the correct location + # in our chroot we need to set PREFIX to <destdir>/<prefix> + # in the configure-commands. + # + # The mapping between PREFIX and the final installation + # directories is complex and depends upon the configuration + # of perl see, + # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories + # and ExtUtil::MakeMaker's documentation for more details. + configure: | + + perl Makefile.PL PREFIX=%{install-root}%{prefix} + + make: make + make-install: make install + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{configure} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} diff --git a/src/buildstream/plugins/elements/manual.py b/src/buildstream/plugins/elements/manual.py new file mode 100644 index 000000000..bbda65312 --- /dev/null +++ b/src/buildstream/plugins/elements/manual.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +manual - Manual build element +============================= +The most basic build element does nothing but allows users to +add custom build commands to the array understood by the :mod:`BuildElement <buildstream.buildelement>` + +The empty configuration is as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/manual.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'manual' kind. +class ManualElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return ManualElement diff --git a/src/buildstream/plugins/elements/manual.yaml b/src/buildstream/plugins/elements/manual.yaml new file mode 100644 index 000000000..38fe7d163 --- /dev/null +++ b/src/buildstream/plugins/elements/manual.yaml @@ -0,0 +1,22 @@ +# Manual build element does not provide any default +# build commands +config: + + # Commands for configuring the software + # + configure-commands: [] + + # Commands for building the software + # + build-commands: [] + + # Commands for installing the software into a + # destination folder + # + install-commands: [] + + # Commands for stripping installed binaries + # + strip-commands: + - | + %{strip-binaries} diff --git a/src/buildstream/plugins/elements/meson.py b/src/buildstream/plugins/elements/meson.py new file mode 100644 index 000000000..d80f77977 --- /dev/null +++ b/src/buildstream/plugins/elements/meson.py @@ -0,0 +1,71 @@ +# Copyright (C) 2017 Patrick Griffis +# Copyright (C) 2018 Codethink Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +meson - Meson build element +=========================== +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using `Meson <http://mesonbuild.com/>`_ build scripts. + +You will often want to pass additional arguments to ``meson``. This should +be done on a per-element basis by setting the ``meson-local`` variable. Here is +an example: + +.. code:: yaml + + variables: + meson-local: | + -Dmonkeys=yes + +If you want to pass extra options to ``meson`` for every element in your +project, set the ``meson-global`` variable in your project.conf file. Here is +an example of that: + +.. code:: yaml + + elements: + meson: + variables: + meson-global: | + -Dmonkeys=always + +Here is the default configuration for the ``meson`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/meson.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'meson' kind. +class MesonElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return MesonElement diff --git a/src/buildstream/plugins/elements/meson.yaml b/src/buildstream/plugins/elements/meson.yaml new file mode 100644 index 000000000..2172cb34c --- /dev/null +++ b/src/buildstream/plugins/elements/meson.yaml @@ -0,0 +1,79 @@ +# Meson default configuration + +variables: + + build-dir: _builddir + + # Project-wide extra arguments to be passed to `meson` + meson-global: '' + + # Element-specific extra arguments to be passed to `meson`. + meson-local: '' + + # For backwards compatibility only, do not use. + meson-extra: '' + + meson-args: | + + --prefix=%{prefix} \ + --bindir=%{bindir} \ + --sbindir=%{sbindir} \ + --sysconfdir=%{sysconfdir} \ + --datadir=%{datadir} \ + --includedir=%{includedir} \ + --libdir=%{libdir} \ + --libexecdir=%{libexecdir} \ + --localstatedir=%{localstatedir} \ + --sharedstatedir=%{sharedstatedir} \ + --mandir=%{mandir} \ + --infodir=%{infodir} %{meson-extra} %{meson-global} %{meson-local} + + meson: meson %{conf-root} %{build-dir} %{meson-args} + + ninja: | + ninja -j ${NINJAJOBS} -C %{build-dir} + + ninja-install: | + env DESTDIR="%{install-root}" ninja -C %{build-dir} install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{meson} + + # Commands for building the software + # + build-commands: + - | + %{ninja} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{ninja-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building +environment: + NINJAJOBS: | + %{max-jobs} + +# And dont consider NINJAJOBS as something which may +# affect build output. +environment-nocache: +- NINJAJOBS diff --git a/src/buildstream/plugins/elements/modulebuild.py b/src/buildstream/plugins/elements/modulebuild.py new file mode 100644 index 000000000..63e3840dc --- /dev/null +++ b/src/buildstream/plugins/elements/modulebuild.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +modulebuild - Perl Module::Build build element +============================================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +the Perl Module::Build build system + +The modulebuild default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/modulebuild.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'modulebuild' kind. +class ModuleBuildElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return ModuleBuildElement diff --git a/src/buildstream/plugins/elements/modulebuild.yaml b/src/buildstream/plugins/elements/modulebuild.yaml new file mode 100644 index 000000000..18f034bab --- /dev/null +++ b/src/buildstream/plugins/elements/modulebuild.yaml @@ -0,0 +1,48 @@ +# Default configuration for the Perl Module::Build +# build system. + +variables: + + # To install perl distributions into the correct location + # in our chroot we need to set PREFIX to <destdir>/<prefix> + # in the configure-commands. + # + # The mapping between PREFIX and the final installation + # directories is complex and depends upon the configuration + # of perl see, + # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories + # and ExtUtil::MakeMaker's documentation for more details. + configure: | + + perl Build.PL --prefix "%{install-root}%{prefix}" + + perl-build: ./Build + perl-install: ./Build install + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{configure} + + # Commands for building the software + # + build-commands: + - | + %{perl-build} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{perl-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} diff --git a/src/buildstream/plugins/elements/pip.py b/src/buildstream/plugins/elements/pip.py new file mode 100644 index 000000000..4a9eefde1 --- /dev/null +++ b/src/buildstream/plugins/elements/pip.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2017 Mathieu Bridon +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Mathieu Bridon <bochecha@daitauha.fr> + +""" +pip - Pip build element +======================= +A :mod:`BuildElement <buildstream.buildelement>` implementation for installing +Python modules with pip + +The pip default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/pip.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'pip' kind. +class PipElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return PipElement diff --git a/src/buildstream/plugins/elements/pip.yaml b/src/buildstream/plugins/elements/pip.yaml new file mode 100644 index 000000000..294d4ad9a --- /dev/null +++ b/src/buildstream/plugins/elements/pip.yaml @@ -0,0 +1,36 @@ +# Pip default configurations + +variables: + + pip: pip + pip-flags: | + %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} + pip-install-package: | + %{pip-flags} %{conf-root} + pip-download-dir: | + .bst_pip_downloads + pip-install-dependencies: | + if [ -e %{pip-download-dir} ]; then %{pip-flags} %{pip-download-dir}/*; fi + +config: + + configure-commands: [] + build-commands: [] + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{pip-install-package} + - | + %{pip-install-dependencies} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + - | + %{fix-pyc-timestamps} diff --git a/src/buildstream/plugins/elements/qmake.py b/src/buildstream/plugins/elements/qmake.py new file mode 100644 index 000000000..56a0e641e --- /dev/null +++ b/src/buildstream/plugins/elements/qmake.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +qmake - QMake build element +=========================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +the qmake build system + +The qmake default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/qmake.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'qmake' kind. +class QMakeElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return QMakeElement diff --git a/src/buildstream/plugins/elements/qmake.yaml b/src/buildstream/plugins/elements/qmake.yaml new file mode 100644 index 000000000..4ac31932e --- /dev/null +++ b/src/buildstream/plugins/elements/qmake.yaml @@ -0,0 +1,50 @@ +# QMake default configuration + +variables: + + qmake: qmake -makefile %{conf-root} + make: make + make-install: make -j1 INSTALL_ROOT="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{qmake} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# affect build output. +environment-nocache: +- MAKEFLAGS +- V diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py new file mode 100644 index 000000000..0d194dcc1 --- /dev/null +++ b/src/buildstream/plugins/elements/script.py @@ -0,0 +1,69 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# Jonathan Maw <jonathan.maw@codethink.co.uk> + +""" +script - Run scripts to create output +===================================== +This element allows one to run some commands to mutate the +input and create some output. + +.. note:: + + Script elements may only specify build dependencies. See + :ref:`the format documentation <format_dependencies>` for more + detail on specifying dependencies. + +The default configuration and possible options are as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/script.yaml + :language: yaml +""" + +import buildstream + + +# Element implementation for the 'script' kind. +class ScriptElement(buildstream.ScriptElement): + # pylint: disable=attribute-defined-outside-init + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + def configure(self, node): + for n in self.node_get_member(node, list, 'layout', []): + dst = self.node_subst_member(n, 'destination') + elm = self.node_subst_member(n, 'element', None) + self.layout_add(elm, dst) + + self.node_validate(node, [ + 'commands', 'root-read-only', 'layout' + ]) + + cmds = self.node_subst_list(node, "commands") + self.add_commands("commands", cmds) + + self.set_work_dir() + self.set_install_root() + self.set_root_read_only(self.node_get_member(node, bool, + 'root-read-only', False)) + + +# Plugin entry point +def setup(): + return ScriptElement diff --git a/src/buildstream/plugins/elements/script.yaml b/src/buildstream/plugins/elements/script.yaml new file mode 100644 index 000000000..b388378da --- /dev/null +++ b/src/buildstream/plugins/elements/script.yaml @@ -0,0 +1,25 @@ +# Common script element variables +variables: + # Defines the directory commands will be run from. + cwd: / + +# Script element configuration +config: + + # Defines whether to run the sandbox with '/' read-only. + # It is recommended to set root as read-only wherever possible. + root-read-only: False + + # Defines where to stage elements which are direct or indirect dependencies. + # By default, all direct dependencies are staged to '/'. + # This is also commonly used to take one element as an environment + # containing the tools used to operate on the other element. + # layout: + # - element: foo-tools.bst + # destination: / + # - element: foo-system.bst + # destination: %{build-root} + + # List of commands to run in the sandbox. + commands: [] + diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py new file mode 100644 index 000000000..97517ca48 --- /dev/null +++ b/src/buildstream/plugins/elements/stack.py @@ -0,0 +1,66 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +stack - Symbolic Element for dependency grouping +================================================ +Stack elements are simply a symbolic element used for representing +a logical group of elements. +""" + +from buildstream import Element + + +# Element implementation for the 'stack' kind. +class StackElement(Element): + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + # We do not add anything to the build, only our dependencies + # do, so our unique key is just a constant. + return 1 + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + + # Just create a dummy empty artifact, its existence is a statement + # that all this stack's dependencies are built. + vrootdir = sandbox.get_virtual_directory() + vrootdir.descend('output', create=True) + + # And we're done + return '/output' + + +# Plugin entry point +def setup(): + return StackElement |