diff options
| author | Chandan Singh <csingh43@bloomberg.net> | 2019-04-24 22:53:19 +0100 |
|---|---|---|
| committer | Chandan Singh <csingh43@bloomberg.net> | 2019-05-21 12:41:18 +0100 |
| commit | 070d053e5cc47e572e9f9e647315082bd7a15c63 (patch) | |
| tree | 7fb0fdff52f9b5f8a18ec8fe9c75b661f9e0839e /src/buildstream/plugins | |
| parent | 6c59e7901a52be961c2a1b671cf2b30f90bc4d0a (diff) | |
| download | buildstream-070d053e5cc47e572e9f9e647315082bd7a15c63.tar.gz | |
Move source from 'buildstream' to 'src/buildstream'
This was discussed in #1008.
Fixes #1009.
Diffstat (limited to 'src/buildstream/plugins')
42 files changed, 3891 insertions, 0 deletions
diff --git a/src/buildstream/plugins/elements/__init__.py b/src/buildstream/plugins/elements/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/buildstream/plugins/elements/__init__.py diff --git a/src/buildstream/plugins/elements/autotools.py b/src/buildstream/plugins/elements/autotools.py new file mode 100644 index 000000000..7a05336b7 --- /dev/null +++ b/src/buildstream/plugins/elements/autotools.py @@ -0,0 +1,75 @@ +# +# Copyright (C) 2016, 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +autotools - Autotools build element +=================================== +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using Autotools build scripts (also known as the `GNU Build System +<https://en.wikipedia.org/wiki/GNU_Build_System>`_). + +You will often want to pass additional arguments to ``configure``. This should +be done on a per-element basis by setting the ``conf-local`` variable. Here is +an example: + +.. code:: yaml + + variables: + conf-local: | + --disable-foo --enable-bar + +If you want to pass extra options to ``configure`` for every element in your +project, set the ``conf-global`` variable in your project.conf file. Here is +an example of that: + +.. code:: yaml + + elements: + autotools: + variables: + conf-global: | + --disable-gtk-doc --disable-static + +Here is the default configuration for the ``autotools`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/autotools.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'autotools' kind. +class AutotoolsElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return AutotoolsElement diff --git a/src/buildstream/plugins/elements/autotools.yaml b/src/buildstream/plugins/elements/autotools.yaml new file mode 100644 index 000000000..85f7393e7 --- /dev/null +++ b/src/buildstream/plugins/elements/autotools.yaml @@ -0,0 +1,129 @@ +# Autotools default configurations + +variables: + + autogen: | + export NOCONFIGURE=1; + + if [ -x %{conf-cmd} ]; then true; + elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen; + elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh; + elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap; + elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh; + else autoreconf -ivf %{conf-root}; + fi + + # Project-wide extra arguments to be passed to `configure` + conf-global: '' + + # Element-specific extra arguments to be passed to `configure`. + conf-local: '' + + # For backwards compatibility only, do not use. + conf-extra: '' + + conf-cmd: "%{conf-root}/configure" + + conf-args: | + + --prefix=%{prefix} \ + --exec-prefix=%{exec_prefix} \ + --bindir=%{bindir} \ + --sbindir=%{sbindir} \ + --sysconfdir=%{sysconfdir} \ + --datadir=%{datadir} \ + --includedir=%{includedir} \ + --libdir=%{libdir} \ + --libexecdir=%{libexecdir} \ + --localstatedir=%{localstatedir} \ + --sharedstatedir=%{sharedstatedir} \ + --mandir=%{mandir} \ + --infodir=%{infodir} %{conf-extra} %{conf-global} %{conf-local} + + configure: | + + %{conf-cmd} %{conf-args} + + make: make + make-install: make -j1 DESTDIR="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + + + # Automatically remove libtool archive files + # + # Set remove-libtool-modules to "true" to remove .la files for + # modules intended to be opened with lt_dlopen() + # + # Set remove-libtool-libraries to "true" to remove .la files for + # libraries + # + # Value must be "true" or "false" + remove-libtool-modules: "false" + remove-libtool-libraries: "false" + + delete-libtool-archives: | + if %{remove-libtool-modules} || %{remove-libtool-libraries}; then + find "%{install-root}" -name "*.la" -print0 | while read -d '' -r file; do + if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then + if %{remove-libtool-modules}; then + echo "Removing ${file}." + rm "${file}" + else + echo "Not removing ${file}." + fi + else + if %{remove-libtool-libraries}; then + echo "Removing ${file}." + rm "${file}" + else + echo "Not removing ${file}." + fi + fi + done + fi + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{autogen} + - | + %{configure} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + - | + %{delete-libtool-archives} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# affect build output. +environment-nocache: +- MAKEFLAGS +- V diff --git a/src/buildstream/plugins/elements/cmake.py b/src/buildstream/plugins/elements/cmake.py new file mode 100644 index 000000000..74da04899 --- /dev/null +++ b/src/buildstream/plugins/elements/cmake.py @@ -0,0 +1,74 @@ +# +# Copyright (C) 2016, 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +cmake - CMake build element +=========================== +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using the `CMake <https://cmake.org/>`_ build system. + +You will often want to pass additional arguments to the ``cmake`` program for +specific configuration options. This should be done on a per-element basis by +setting the ``cmake-local`` variable. Here is an example: + +.. code:: yaml + + variables: + cmake-local: | + -DCMAKE_BUILD_TYPE=Debug + +If you want to pass extra options to ``cmake`` for every element in your +project, set the ``cmake-global`` variable in your project.conf file. Here is +an example of that: + +.. code:: yaml + + elements: + cmake: + variables: + cmake-global: | + -DCMAKE_BUILD_TYPE=Release + +Here is the default configuration for the ``cmake`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/cmake.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'cmake' kind. +class CMakeElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return CMakeElement diff --git a/src/buildstream/plugins/elements/cmake.yaml b/src/buildstream/plugins/elements/cmake.yaml new file mode 100644 index 000000000..ba20d7ce6 --- /dev/null +++ b/src/buildstream/plugins/elements/cmake.yaml @@ -0,0 +1,72 @@ +# CMake default configuration + +variables: + + build-dir: _builddir + + # Project-wide extra arguments to be passed to `cmake` + cmake-global: '' + + # Element-specific extra arguments to be passed to `cmake`. + cmake-local: '' + + # For backwards compatibility only, do not use. + cmake-extra: '' + + # The cmake generator to use + generator: Unix Makefiles + + cmake-args: | + + -DCMAKE_INSTALL_PREFIX:PATH="%{prefix}" \ + -DCMAKE_INSTALL_LIBDIR:PATH="%{lib}" %{cmake-extra} %{cmake-global} %{cmake-local} + + cmake: | + + cmake -B%{build-dir} -H"%{conf-root}" -G"%{generator}" %{cmake-args} + + make: cmake --build %{build-dir} -- ${JOBS} + make-install: env DESTDIR="%{install-root}" cmake --build %{build-dir} --target install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{cmake} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + JOBS: -j%{max-jobs} + V: 1 + +# And dont consider JOBS or V as something which may +# affect build output. +environment-nocache: +- JOBS +- V diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py new file mode 100644 index 000000000..b672cde0c --- /dev/null +++ b/src/buildstream/plugins/elements/compose.py @@ -0,0 +1,194 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +compose - Compose the output of multiple elements +================================================= +This element creates a selective composition of its dependencies. + +This is normally used at near the end of a pipeline to prepare +something for later deployment. + +Since this element's output includes its dependencies, it may only +depend on elements as `build` type dependencies. + +The default configuration and possible options are as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/compose.yaml + :language: yaml +""" + +import os +from buildstream import Element, Scope + + +# Element implementation for the 'compose' kind. +class ComposeElement(Element): + # pylint: disable=attribute-defined-outside-init + + # The compose element's output is its dependencies, so + # we must rebuild if the dependencies change even when + # not in strict build plans. + # + BST_STRICT_REBUILD = True + + # Compose artifacts must never have indirect dependencies, + # so runtime dependencies are forbidden. + BST_FORBID_RDEPENDS = True + + # This element ignores sources, so we should forbid them from being + # added, to reduce the potential for confusion + BST_FORBID_SOURCES = True + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + def configure(self, node): + self.node_validate(node, [ + 'integrate', 'include', 'exclude', 'include-orphans' + ]) + + # We name this variable 'integration' only to avoid + # collision with the Element.integrate() method. + self.integration = self.node_get_member(node, bool, 'integrate') + self.include = self.node_get_member(node, list, 'include') + self.exclude = self.node_get_member(node, list, 'exclude') + self.include_orphans = self.node_get_member(node, bool, 'include-orphans') + + def preflight(self): + pass + + def get_unique_key(self): + key = {'integrate': self.integration, + 'include': sorted(self.include), + 'orphans': self.include_orphans} + + if self.exclude: + key['exclude'] = sorted(self.exclude) + + return key + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + + require_split = self.include or self.exclude or not self.include_orphans + + # Stage deps in the sandbox root + with self.timed_activity("Staging dependencies", silent_nested=True): + self.stage_dependency_artifacts(sandbox, Scope.BUILD) + + manifest = set() + if require_split: + with self.timed_activity("Computing split", silent_nested=True): + for dep in self.dependencies(Scope.BUILD): + files = dep.compute_manifest(include=self.include, + exclude=self.exclude, + orphans=self.include_orphans) + manifest.update(files) + + # Make a snapshot of all the files. + vbasedir = sandbox.get_virtual_directory() + modified_files = set() + removed_files = set() + added_files = set() + + # Run any integration commands provided by the dependencies + # once they are all staged and ready + if self.integration: + with self.timed_activity("Integrating sandbox"): + if require_split: + + # Make a snapshot of all the files before integration-commands are run. + snapshot = set(vbasedir.list_relative_paths()) + vbasedir.mark_unmodified() + + with sandbox.batch(0): + for dep in self.dependencies(Scope.BUILD): + dep.integrate(sandbox) + + if require_split: + # Calculate added, modified and removed files + post_integration_snapshot = vbasedir.list_relative_paths() + modified_files = set(vbasedir.list_modified_paths()) + basedir_contents = set(post_integration_snapshot) + for path in manifest: + if path in snapshot and path not in basedir_contents: + removed_files.add(path) + + for path in basedir_contents: + if path not in snapshot: + added_files.add(path) + self.info("Integration modified {}, added {} and removed {} files" + .format(len(modified_files), len(added_files), len(removed_files))) + + # The remainder of this is expensive, make an early exit if + # we're not being selective about what is to be included. + if not require_split: + return '/' + + # Do we want to force include files which were modified by + # the integration commands, even if they were not added ? + # + manifest.update(added_files) + manifest.difference_update(removed_files) + + # XXX We should be moving things outside of the build sandbox + # instead of into a subdir. The element assemble() method should + # support this in some way. + # + installdir = vbasedir.descend('buildstream', 'install', create=True) + + # We already saved the manifest for created files in the integration phase, + # now collect the rest of the manifest. + # + + lines = [] + if self.include: + lines.append("Including files from domains: " + ", ".join(self.include)) + else: + lines.append("Including files from all domains") + + if self.exclude: + lines.append("Excluding files from domains: " + ", ".join(self.exclude)) + + if self.include_orphans: + lines.append("Including orphaned files") + else: + lines.append("Excluding orphaned files") + + detail = "\n".join(lines) + + def import_filter(path): + return path in manifest + + with self.timed_activity("Creating composition", detail=detail, silent_nested=True): + self.info("Composing {} files".format(len(manifest))) + installdir.import_files(vbasedir, filter_callback=import_filter, can_link=True) + + # And we're done + return os.path.join(os.sep, 'buildstream', 'install') + + +# Plugin entry point +def setup(): + return ComposeElement diff --git a/src/buildstream/plugins/elements/compose.yaml b/src/buildstream/plugins/elements/compose.yaml new file mode 100644 index 000000000..fd2eb9358 --- /dev/null +++ b/src/buildstream/plugins/elements/compose.yaml @@ -0,0 +1,34 @@ + +# Compose element configuration +config: + + # Whether to run the integration commands for the + # staged dependencies. + # + integrate: True + + # A list of domains to include from each artifact, as + # they were defined in the element's 'split-rules'. + # + # Since domains can be added, it is not an error to + # specify domains which may not exist for all of the + # elements in this composition. + # + # The default empty list indicates that all domains + # from each dependency should be included. + # + include: [] + + # A list of domains to exclude from each artifact, as + # they were defined in the element's 'split-rules'. + # + # In the case that a file is spoken for by a domain + # in the 'include' list and another in the 'exclude' + # list, then the file will be excluded. + exclude: [] + + # Whether to include orphan files which are not + # included by any of the 'split-rules' present on + # a given element. + # + include-orphans: True diff --git a/src/buildstream/plugins/elements/distutils.py b/src/buildstream/plugins/elements/distutils.py new file mode 100644 index 000000000..4b2c1e2f4 --- /dev/null +++ b/src/buildstream/plugins/elements/distutils.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +distutils - Python distutils element +==================================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +python distutils + +The distutils default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/distutils.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the python 'distutils' kind. +class DistutilsElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return DistutilsElement diff --git a/src/buildstream/plugins/elements/distutils.yaml b/src/buildstream/plugins/elements/distutils.yaml new file mode 100644 index 000000000..cec7da6e9 --- /dev/null +++ b/src/buildstream/plugins/elements/distutils.yaml @@ -0,0 +1,49 @@ +# Default python distutils configuration + +variables: + + # When building for python2 distutils, simply + # override this in the element declaration + python: python3 + + python-build: | + + %{python} %{conf-root}/setup.py build + + install-args: | + + --prefix "%{prefix}" \ + --root "%{install-root}" + + python-install: | + + %{python} %{conf-root}/setup.py install %{install-args} + + +config: + + # Commands for configuring the software + # + configure-commands: [] + + # Commands for building the software + # + build-commands: + - | + %{python-build} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{python-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + - | + %{fix-pyc-timestamps} diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py new file mode 100644 index 000000000..45847e685 --- /dev/null +++ b/src/buildstream/plugins/elements/filter.py @@ -0,0 +1,256 @@ +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jonathan Maw <jonathan.maw@codethink.co.uk> + +""" +filter - Extract a subset of files from another element +======================================================= +Filter another element by producing an output that is a subset of +the parent element's output. Subsets are defined by the parent element's +:ref:`split rules <public_split_rules>`. + +Overview +-------- +A filter element must have exactly one *build* dependency, where said +dependency is the 'parent' element which we would like to filter. +Runtime dependencies may also be specified, which can be useful to propagate +forward from this filter element onto its reverse dependencies. +See :ref:`Dependencies <format_dependencies>` to see how we specify dependencies. + +When workspaces are opened, closed or reset on a filter element, or this +element is tracked, the filter element will transparently pass on the command +to its parent element (the sole build-dependency). + +Example +------- +Consider a simple import element, ``import.bst`` which imports the local files +'foo', 'bar' and 'baz' (each stored in ``files/``, relative to the project's root): + +.. code:: yaml + + kind: import + + # Specify sources to import + sources: + - kind: local + path: files + + # Specify public domain data, visible to other elements + public: + bst: + split-rules: + foo: + - /foo + bar: + - /bar + +.. note:: + + We can make an element's metadata visible to all reverse dependencies by making use + of the ``public:`` field. See the :ref:`public data documentation <format_public>` + for more information. + +In this example, ``import.bst`` will serve as the 'parent' of the filter element, thus +its output will be filtered. It is important to understand that the artifact of the +above element will contain the files: 'foo', 'bar' and 'baz'. + +Now, to produce an element whose artifact contains the file 'foo', and exlusively 'foo', +we can define the following filter, ``filter-foo.bst``: + +.. code:: yaml + + kind: filter + + # Declare the sole build-dependency of the filter element + depends: + - filename: import.bst + type: build + + # Declare a list of domains to include in the filter's artifact + config: + include: + - foo + +.. note:: + + We can also specify build-dependencies with a 'build-depends' field which has been + available since :ref:`format version 14 <project_format_version>`. See the + :ref:`Build-Depends documentation <format_build_depends>` for more detail. + +It should be noted that an 'empty' ``include:`` list would, by default, include all +split-rules specified in the parent element, which, in this example, would be the +files 'foo' and 'bar' (the file 'baz' was not covered by any split rules). + +Equally, we can use the ``exclude:`` statement to create the same artifact (which +only contains the file 'foo') by declaring the following element, ``exclude-bar.bst``: + +.. code:: yaml + + kind: filter + + # Declare the sole build-dependency of the filter element + depends: + - filename: import.bst + type: build + + # Declare a list of domains to exclude in the filter's artifact + config: + exclude: + - bar + +In addition to the ``include:`` and ``exclude:`` fields, there exists an ``include-orphans:`` +(Boolean) field, which defaults to ``False``. This will determine whether to include files +which are not present in the 'split-rules'. For example, if we wanted to filter out all files +which are not included as split rules we can define the following element, ``filter-misc.bst``: + +.. code:: yaml + + kind: filter + + # Declare the sole build-dependency of the filter element + depends: + - filename: import.bst + type: build + + # Filter out all files which are not declared as split rules + config: + exclude: + - foo + - bar + include-orphans: True + +The artifact of ``filter-misc.bst`` will only contain the file 'baz'. + +Below is more information regarding the the default configurations and possible options +of the filter element: + +.. literalinclude:: ../../../src/buildstream/plugins/elements/filter.yaml + :language: yaml +""" + +from buildstream import Element, ElementError, Scope + + +class FilterElement(Element): + # pylint: disable=attribute-defined-outside-init + + BST_ARTIFACT_VERSION = 1 + + # The filter element's output is its dependencies, so + # we must rebuild if the dependencies change even when + # not in strict build plans. + BST_STRICT_REBUILD = True + + # This element ignores sources, so we should forbid them from being + # added, to reduce the potential for confusion + BST_FORBID_SOURCES = True + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + # Filter elements do not run any commands + BST_RUN_COMMANDS = False + + def configure(self, node): + self.node_validate(node, [ + 'include', 'exclude', 'include-orphans' + ]) + + self.include = self.node_get_member(node, list, 'include') + self.exclude = self.node_get_member(node, list, 'exclude') + self.include_orphans = self.node_get_member(node, bool, 'include-orphans') + self.include_provenance = self.node_provenance(node, member_name='include') + self.exclude_provenance = self.node_provenance(node, member_name='exclude') + + def preflight(self): + # Exactly one build-depend is permitted + build_deps = list(self.dependencies(Scope.BUILD, recurse=False)) + if len(build_deps) != 1: + detail = "Full list of build-depends:\n" + deps_list = " \n".join([x.name for x in build_deps]) + detail += deps_list + raise ElementError("{}: {} element must have exactly 1 build-dependency, actually have {}" + .format(self, type(self).__name__, len(build_deps)), + detail=detail, reason="filter-bdepend-wrong-count") + + # That build-depend must not also be a runtime-depend + runtime_deps = list(self.dependencies(Scope.RUN, recurse=False)) + if build_deps[0] in runtime_deps: + detail = "Full list of runtime depends:\n" + deps_list = " \n".join([x.name for x in runtime_deps]) + detail += deps_list + raise ElementError("{}: {} element's build dependency must not also be a runtime dependency" + .format(self, type(self).__name__), + detail=detail, reason="filter-bdepend-also-rdepend") + + def get_unique_key(self): + key = { + 'include': sorted(self.include), + 'exclude': sorted(self.exclude), + 'orphans': self.include_orphans, + } + return key + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + with self.timed_activity("Staging artifact", silent_nested=True): + for dep in self.dependencies(Scope.BUILD, recurse=False): + # Check that all the included/excluded domains exist + pub_data = dep.get_public_data('bst') + split_rules = self.node_get_member(pub_data, dict, 'split-rules', {}) + unfound_includes = [] + for domain in self.include: + if domain not in split_rules: + unfound_includes.append(domain) + unfound_excludes = [] + for domain in self.exclude: + if domain not in split_rules: + unfound_excludes.append(domain) + + detail = [] + if unfound_includes: + detail.append("Unknown domains were used in {}".format(self.include_provenance)) + detail.extend([' - {}'.format(domain) for domain in unfound_includes]) + + if unfound_excludes: + detail.append("Unknown domains were used in {}".format(self.exclude_provenance)) + detail.extend([' - {}'.format(domain) for domain in unfound_excludes]) + + if detail: + detail = '\n'.join(detail) + raise ElementError("Unknown domains declared.", detail=detail) + + dep.stage_artifact(sandbox, include=self.include, + exclude=self.exclude, orphans=self.include_orphans) + return "" + + def _get_source_element(self): + # Filter elements act as proxies for their sole build-dependency + build_deps = list(self.dependencies(Scope.BUILD, recurse=False)) + assert len(build_deps) == 1 + output_elm = build_deps[0]._get_source_element() + return output_elm + + +def setup(): + return FilterElement diff --git a/src/buildstream/plugins/elements/filter.yaml b/src/buildstream/plugins/elements/filter.yaml new file mode 100644 index 000000000..9c2bf69f4 --- /dev/null +++ b/src/buildstream/plugins/elements/filter.yaml @@ -0,0 +1,29 @@ + +# Filter element configuration +config: + + # A list of domains to include in each artifact, as + # they were defined as public data in the parent + # element's 'split-rules'. + # + # If a domain is specified that does not exist, the + # filter element will fail to build. + # + # The default empty list indicates that all domains + # of the parent's artifact should be included. + # + include: [] + + # A list of domains to exclude from each artifact, as + # they were defined in the parent element's 'split-rules'. + # + # In the case that a file is spoken for by a domain + # in the 'include' list and another in the 'exclude' + # list, then the file will be excluded. + exclude: [] + + # Whether to include orphan files which are not + # included by any of the 'split-rules' present in + # the parent element. + # + include-orphans: False diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py new file mode 100644 index 000000000..61e353dbc --- /dev/null +++ b/src/buildstream/plugins/elements/import.py @@ -0,0 +1,129 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +import - Import sources directly +================================ +Import elements produce artifacts directly from its sources +without any kind of processing. These are typically used to +import an SDK to build on top of or to overlay your build with +some configuration data. + +The empty configuration is as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/import.yaml + :language: yaml +""" + +import os +from buildstream import Element, ElementError + + +# Element implementation for the 'import' kind. +class ImportElement(Element): + # pylint: disable=attribute-defined-outside-init + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + # Import elements do not run any commands + BST_RUN_COMMANDS = False + + def configure(self, node): + self.node_validate(node, [ + 'source', 'target' + ]) + + self.source = self.node_subst_member(node, 'source') + self.target = self.node_subst_member(node, 'target') + + def preflight(self): + # Assert that we have at least one source to fetch. + + sources = list(self.sources()) + if not sources: + raise ElementError("{}: An import element must have at least one source.".format(self)) + + def get_unique_key(self): + return { + 'source': self.source, + 'target': self.target + } + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + + # Stage sources into the input directory + # Do not mount workspaces as the files are copied from outside the sandbox + self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False) + + rootdir = sandbox.get_virtual_directory() + inputdir = rootdir.descend('input') + outputdir = rootdir.descend('output', create=True) + + # The directory to grab + inputdir = inputdir.descend(*self.source.strip(os.sep).split(os.sep)) + + # The output target directory + outputdir = outputdir.descend(*self.target.strip(os.sep).split(os.sep), create=True) + + if inputdir.is_empty(): + raise ElementError("{}: No files were found inside directory '{}'" + .format(self, self.source)) + + # Move it over + outputdir.import_files(inputdir) + + # And we're done + return '/output' + + def generate_script(self): + build_root = self.get_variable('build-root') + install_root = self.get_variable('install-root') + commands = [] + + # The directory to grab + inputdir = os.path.join(build_root, self.normal_name, self.source.lstrip(os.sep)) + inputdir = inputdir.rstrip(os.sep) + + # The output target directory + outputdir = os.path.join(install_root, self.target.lstrip(os.sep)) + outputdir = outputdir.rstrip(os.sep) + + # Ensure target directory parent exists but target directory doesn't + commands.append("mkdir -p {}".format(os.path.dirname(outputdir))) + commands.append("[ ! -e {outputdir} ] || rmdir {outputdir}".format(outputdir=outputdir)) + + # Move it over + commands.append("mv {} {}".format(inputdir, outputdir)) + + script = "" + for cmd in commands: + script += "(set -ex; {}\n) || exit 1\n".format(cmd) + + return script + + +# Plugin entry point +def setup(): + return ImportElement diff --git a/src/buildstream/plugins/elements/import.yaml b/src/buildstream/plugins/elements/import.yaml new file mode 100644 index 000000000..698111b55 --- /dev/null +++ b/src/buildstream/plugins/elements/import.yaml @@ -0,0 +1,14 @@ +# The import element simply stages the given sources +# directly to the root of the sandbox and then collects +# the output to create an output artifact. +# +config: + + # By default we collect everything staged, specify a + # directory here to output only a subset of the staged + # input sources. + source: / + + # Prefix the output with an optional directory, by default + # the input is found at the root of the produced artifact. + target: / diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py new file mode 100644 index 000000000..15ef115d9 --- /dev/null +++ b/src/buildstream/plugins/elements/junction.py @@ -0,0 +1,229 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jürg Billeter <juerg.billeter@codethink.co.uk> + +""" +junction - Integrate subprojects +================================ +This element is a link to another BuildStream project. It allows integration +of multiple projects into a single pipeline. + +Overview +-------- + +.. code:: yaml + + kind: junction + + # Specify the BuildStream project source + sources: + - kind: git + url: upstream:projectname.git + track: master + ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6 + + # Specify the junction configuration + config: + + # Override project options + options: + machine_arch: "%{machine_arch}" + debug: True + + # Optionally look in a subpath of the source repository for the project + path: projects/hello + + # Optionally specify another junction element to serve as a target for + # this element. Target should be defined using the syntax + # ``{junction-name}:{element-name}``. + # + # Note that this option cannot be used in conjunction with sources. + target: sub-project.bst:sub-sub-project.bst + +.. note:: + + The configuration option to allow specifying junction targets is available + since :ref:`format version 24 <project_format_version>`. + +.. note:: + + Junction elements may not specify any dependencies as they are simply + links to other projects and are not in the dependency graph on their own. + +With a junction element in place, local elements can depend on elements in +the other BuildStream project using the additional ``junction`` attribute in the +dependency dictionary: + +.. code:: yaml + + depends: + - junction: toolchain.bst + filename: gcc.bst + type: build + +While junctions are elements, only a limited set of element operations is +supported. They can be tracked and fetched like other elements. +However, junction elements do not produce any artifacts, which means that +they cannot be built or staged. It also means that another element cannot +depend on a junction element itself. + +.. note:: + + BuildStream does not implicitly track junction elements. This means + that if we were to invoke: `bst build --track-all ELEMENT` on an element + which uses a junction element, the ref of the junction element + will not automatically be updated if a more recent version exists. + + Therefore, if you require the most up-to-date version of a subproject, + you must explicitly track the junction element by invoking: + `bst source track JUNCTION_ELEMENT`. + + Furthermore, elements within the subproject are also not tracked by default. + For this, we must specify the `--track-cross-junctions` option. This option + must be preceeded by `--track ELEMENT` or `--track-all`. + + +Sources +------- +``bst show`` does not implicitly fetch junction sources if they haven't been +cached yet. However, they can be fetched explicitly: + +.. code:: + + bst source fetch junction.bst + +Other commands such as ``bst build`` implicitly fetch junction sources. + +Options +------- +.. code:: yaml + + options: + machine_arch: "%{machine_arch}" + debug: True + +Junctions can configure options of the linked project. Options are never +implicitly inherited across junctions, however, variables can be used to +explicitly assign the same value to a subproject option. + +.. _core_junction_nested: + +Nested Junctions +---------------- +Junctions can be nested. That is, subprojects are allowed to have junctions on +their own. Nested junctions in different subprojects may point to the same +project, however, in most use cases the same project should be loaded only once. +BuildStream uses the junction element name as key to determine which junctions +to merge. It is recommended that the name of a junction is set to the same as +the name of the linked project. + +As the junctions may differ in source version and options, BuildStream cannot +simply use one junction and ignore the others. Due to this, BuildStream requires +the user to resolve possibly conflicting nested junctions by creating a junction +with the same name in the top-level project, which then takes precedence. + +Targeting other junctions +~~~~~~~~~~~~~~~~~~~~~~~~~ +When working with nested junctions, you can also create a junction element that +targets another junction element in the sub-project. This can be useful if you +need to ensure that both the top-level project and the sub-project are using +the same version of the sub-sub-project. + +This can be done using the ``target`` configuration option. See below for an +example: + +.. code:: yaml + + kind: junction + + config: + target: subproject.bst:subsubproject.bst + +In the above example, this junction element would be targeting the junction +element named ``subsubproject.bst`` in the subproject referred to by +``subproject.bst``. + +Note that when targeting another junction, the names of the junction element +must not be the same as the name of the target. +""" + +from collections.abc import Mapping +from buildstream import Element, ElementError +from buildstream._pipeline import PipelineError + + +# Element implementation for the 'junction' kind. +class JunctionElement(Element): + # pylint: disable=attribute-defined-outside-init + + # Junctions are not allowed any dependencies + BST_FORBID_BDEPENDS = True + BST_FORBID_RDEPENDS = True + + def configure(self, node): + self.path = self.node_get_member(node, str, 'path', default='') + self.options = self.node_get_member(node, Mapping, 'options', default={}) + self.target = self.node_get_member(node, str, 'target', default=None) + self.target_element = None + self.target_junction = None + + def preflight(self): + # "target" cannot be used in conjunction with: + # 1. sources + # 2. config['options'] + # 3. config['path'] + if self.target and any(self.sources()): + raise ElementError("junction elements cannot define both 'sources' and 'target' config option") + if self.target and any(self.node_items(self.options)): + raise ElementError("junction elements cannot define both 'options' and 'target'") + if self.target and self.path: + raise ElementError("junction elements cannot define both 'path' and 'target'") + + # Validate format of target, if defined + if self.target: + try: + self.target_junction, self.target_element = self.target.split(":") + except ValueError: + raise ElementError("'target' option must be in format '{junction-name}:{element-name}'") + + # We cannot target a junction that has the same name as us, since that + # will cause an infinite recursion while trying to load it. + if self.name == self.target_element: + raise ElementError("junction elements cannot target an element with the same name") + + def get_unique_key(self): + # Junctions do not produce artifacts. get_unique_key() implementation + # is still required for `bst source fetch`. + return 1 + + def configure_sandbox(self, sandbox): + raise PipelineError("Cannot build junction elements") + + def stage(self, sandbox): + raise PipelineError("Cannot stage junction elements") + + def generate_script(self): + raise PipelineError("Cannot build junction elements") + + def assemble(self, sandbox): + raise PipelineError("Cannot build junction elements") + + +# Plugin entry point +def setup(): + return JunctionElement diff --git a/src/buildstream/plugins/elements/make.py b/src/buildstream/plugins/elements/make.py new file mode 100644 index 000000000..67a261100 --- /dev/null +++ b/src/buildstream/plugins/elements/make.py @@ -0,0 +1,56 @@ +# +# Copyright Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Ed Baunton <ebaunton1@bloomberg.net> + +""" +make - Make build element +========================= +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using GNU make based build. + +.. note:: + + The ``make`` element is available since :ref:`format version 9 <project_format_version>` + +Here is the default configuration for the ``make`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/make.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'make' kind. +class MakeElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return MakeElement diff --git a/src/buildstream/plugins/elements/make.yaml b/src/buildstream/plugins/elements/make.yaml new file mode 100644 index 000000000..83e5c658f --- /dev/null +++ b/src/buildstream/plugins/elements/make.yaml @@ -0,0 +1,42 @@ +# make default configurations + +variables: + make: make PREFIX="%{prefix}" + make-install: make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# affect build output. +environment-nocache: +- MAKEFLAGS +- V diff --git a/src/buildstream/plugins/elements/makemaker.py b/src/buildstream/plugins/elements/makemaker.py new file mode 100644 index 000000000..7da051592 --- /dev/null +++ b/src/buildstream/plugins/elements/makemaker.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +makemaker - Perl MakeMaker build element +======================================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +the Perl ExtUtil::MakeMaker build system + +The MakeMaker default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/makemaker.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'makemaker' kind. +class MakeMakerElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return MakeMakerElement diff --git a/src/buildstream/plugins/elements/makemaker.yaml b/src/buildstream/plugins/elements/makemaker.yaml new file mode 100644 index 000000000..c9c4622cb --- /dev/null +++ b/src/buildstream/plugins/elements/makemaker.yaml @@ -0,0 +1,48 @@ +# Default configuration for the Perl ExtUtil::MakeMaker +# build system + +variables: + + # To install perl distributions into the correct location + # in our chroot we need to set PREFIX to <destdir>/<prefix> + # in the configure-commands. + # + # The mapping between PREFIX and the final installation + # directories is complex and depends upon the configuration + # of perl see, + # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories + # and ExtUtil::MakeMaker's documentation for more details. + configure: | + + perl Makefile.PL PREFIX=%{install-root}%{prefix} + + make: make + make-install: make install + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{configure} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} diff --git a/src/buildstream/plugins/elements/manual.py b/src/buildstream/plugins/elements/manual.py new file mode 100644 index 000000000..bbda65312 --- /dev/null +++ b/src/buildstream/plugins/elements/manual.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +manual - Manual build element +============================= +The most basic build element does nothing but allows users to +add custom build commands to the array understood by the :mod:`BuildElement <buildstream.buildelement>` + +The empty configuration is as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/manual.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'manual' kind. +class ManualElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return ManualElement diff --git a/src/buildstream/plugins/elements/manual.yaml b/src/buildstream/plugins/elements/manual.yaml new file mode 100644 index 000000000..38fe7d163 --- /dev/null +++ b/src/buildstream/plugins/elements/manual.yaml @@ -0,0 +1,22 @@ +# Manual build element does not provide any default +# build commands +config: + + # Commands for configuring the software + # + configure-commands: [] + + # Commands for building the software + # + build-commands: [] + + # Commands for installing the software into a + # destination folder + # + install-commands: [] + + # Commands for stripping installed binaries + # + strip-commands: + - | + %{strip-binaries} diff --git a/src/buildstream/plugins/elements/meson.py b/src/buildstream/plugins/elements/meson.py new file mode 100644 index 000000000..d80f77977 --- /dev/null +++ b/src/buildstream/plugins/elements/meson.py @@ -0,0 +1,71 @@ +# Copyright (C) 2017 Patrick Griffis +# Copyright (C) 2018 Codethink Ltd. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. + +""" +meson - Meson build element +=========================== +This is a :mod:`BuildElement <buildstream.buildelement>` implementation for +using `Meson <http://mesonbuild.com/>`_ build scripts. + +You will often want to pass additional arguments to ``meson``. This should +be done on a per-element basis by setting the ``meson-local`` variable. Here is +an example: + +.. code:: yaml + + variables: + meson-local: | + -Dmonkeys=yes + +If you want to pass extra options to ``meson`` for every element in your +project, set the ``meson-global`` variable in your project.conf file. Here is +an example of that: + +.. code:: yaml + + elements: + meson: + variables: + meson-global: | + -Dmonkeys=always + +Here is the default configuration for the ``meson`` element in full: + + .. literalinclude:: ../../../src/buildstream/plugins/elements/meson.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'meson' kind. +class MesonElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return MesonElement diff --git a/src/buildstream/plugins/elements/meson.yaml b/src/buildstream/plugins/elements/meson.yaml new file mode 100644 index 000000000..2172cb34c --- /dev/null +++ b/src/buildstream/plugins/elements/meson.yaml @@ -0,0 +1,79 @@ +# Meson default configuration + +variables: + + build-dir: _builddir + + # Project-wide extra arguments to be passed to `meson` + meson-global: '' + + # Element-specific extra arguments to be passed to `meson`. + meson-local: '' + + # For backwards compatibility only, do not use. + meson-extra: '' + + meson-args: | + + --prefix=%{prefix} \ + --bindir=%{bindir} \ + --sbindir=%{sbindir} \ + --sysconfdir=%{sysconfdir} \ + --datadir=%{datadir} \ + --includedir=%{includedir} \ + --libdir=%{libdir} \ + --libexecdir=%{libexecdir} \ + --localstatedir=%{localstatedir} \ + --sharedstatedir=%{sharedstatedir} \ + --mandir=%{mandir} \ + --infodir=%{infodir} %{meson-extra} %{meson-global} %{meson-local} + + meson: meson %{conf-root} %{build-dir} %{meson-args} + + ninja: | + ninja -j ${NINJAJOBS} -C %{build-dir} + + ninja-install: | + env DESTDIR="%{install-root}" ninja -C %{build-dir} install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{meson} + + # Commands for building the software + # + build-commands: + - | + %{ninja} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{ninja-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building +environment: + NINJAJOBS: | + %{max-jobs} + +# And dont consider NINJAJOBS as something which may +# affect build output. +environment-nocache: +- NINJAJOBS diff --git a/src/buildstream/plugins/elements/modulebuild.py b/src/buildstream/plugins/elements/modulebuild.py new file mode 100644 index 000000000..63e3840dc --- /dev/null +++ b/src/buildstream/plugins/elements/modulebuild.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +modulebuild - Perl Module::Build build element +============================================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +the Perl Module::Build build system + +The modulebuild default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/modulebuild.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'modulebuild' kind. +class ModuleBuildElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return ModuleBuildElement diff --git a/src/buildstream/plugins/elements/modulebuild.yaml b/src/buildstream/plugins/elements/modulebuild.yaml new file mode 100644 index 000000000..18f034bab --- /dev/null +++ b/src/buildstream/plugins/elements/modulebuild.yaml @@ -0,0 +1,48 @@ +# Default configuration for the Perl Module::Build +# build system. + +variables: + + # To install perl distributions into the correct location + # in our chroot we need to set PREFIX to <destdir>/<prefix> + # in the configure-commands. + # + # The mapping between PREFIX and the final installation + # directories is complex and depends upon the configuration + # of perl see, + # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories + # and ExtUtil::MakeMaker's documentation for more details. + configure: | + + perl Build.PL --prefix "%{install-root}%{prefix}" + + perl-build: ./Build + perl-install: ./Build install + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{configure} + + # Commands for building the software + # + build-commands: + - | + %{perl-build} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{perl-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} diff --git a/src/buildstream/plugins/elements/pip.py b/src/buildstream/plugins/elements/pip.py new file mode 100644 index 000000000..4a9eefde1 --- /dev/null +++ b/src/buildstream/plugins/elements/pip.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2017 Mathieu Bridon +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Mathieu Bridon <bochecha@daitauha.fr> + +""" +pip - Pip build element +======================= +A :mod:`BuildElement <buildstream.buildelement>` implementation for installing +Python modules with pip + +The pip default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/pip.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'pip' kind. +class PipElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return PipElement diff --git a/src/buildstream/plugins/elements/pip.yaml b/src/buildstream/plugins/elements/pip.yaml new file mode 100644 index 000000000..294d4ad9a --- /dev/null +++ b/src/buildstream/plugins/elements/pip.yaml @@ -0,0 +1,36 @@ +# Pip default configurations + +variables: + + pip: pip + pip-flags: | + %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix} + pip-install-package: | + %{pip-flags} %{conf-root} + pip-download-dir: | + .bst_pip_downloads + pip-install-dependencies: | + if [ -e %{pip-download-dir} ]; then %{pip-flags} %{pip-download-dir}/*; fi + +config: + + configure-commands: [] + build-commands: [] + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{pip-install-package} + - | + %{pip-install-dependencies} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + - | + %{fix-pyc-timestamps} diff --git a/src/buildstream/plugins/elements/qmake.py b/src/buildstream/plugins/elements/qmake.py new file mode 100644 index 000000000..56a0e641e --- /dev/null +++ b/src/buildstream/plugins/elements/qmake.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +qmake - QMake build element +=========================== +A :mod:`BuildElement <buildstream.buildelement>` implementation for using +the qmake build system + +The qmake default configuration: + .. literalinclude:: ../../../src/buildstream/plugins/elements/qmake.yaml + :language: yaml + +See :ref:`built-in functionality documentation <core_buildelement_builtins>` for +details on common configuration options for build elements. +""" + +from buildstream import BuildElement, SandboxFlags + + +# Element implementation for the 'qmake' kind. +class QMakeElement(BuildElement): + # Supports virtual directories (required for remote execution) + BST_VIRTUAL_DIRECTORY = True + + # Enable command batching across prepare() and assemble() + def configure_sandbox(self, sandbox): + super().configure_sandbox(sandbox) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, + collect=self.get_variable('install-root')) + + +# Plugin entry point +def setup(): + return QMakeElement diff --git a/src/buildstream/plugins/elements/qmake.yaml b/src/buildstream/plugins/elements/qmake.yaml new file mode 100644 index 000000000..4ac31932e --- /dev/null +++ b/src/buildstream/plugins/elements/qmake.yaml @@ -0,0 +1,50 @@ +# QMake default configuration + +variables: + + qmake: qmake -makefile %{conf-root} + make: make + make-install: make -j1 INSTALL_ROOT="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for configuring the software + # + configure-commands: + - | + %{qmake} + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# affect build output. +environment-nocache: +- MAKEFLAGS +- V diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py new file mode 100644 index 000000000..0d194dcc1 --- /dev/null +++ b/src/buildstream/plugins/elements/script.py @@ -0,0 +1,69 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# Jonathan Maw <jonathan.maw@codethink.co.uk> + +""" +script - Run scripts to create output +===================================== +This element allows one to run some commands to mutate the +input and create some output. + +.. note:: + + Script elements may only specify build dependencies. See + :ref:`the format documentation <format_dependencies>` for more + detail on specifying dependencies. + +The default configuration and possible options are as such: + .. literalinclude:: ../../../src/buildstream/plugins/elements/script.yaml + :language: yaml +""" + +import buildstream + + +# Element implementation for the 'script' kind. +class ScriptElement(buildstream.ScriptElement): + # pylint: disable=attribute-defined-outside-init + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + def configure(self, node): + for n in self.node_get_member(node, list, 'layout', []): + dst = self.node_subst_member(n, 'destination') + elm = self.node_subst_member(n, 'element', None) + self.layout_add(elm, dst) + + self.node_validate(node, [ + 'commands', 'root-read-only', 'layout' + ]) + + cmds = self.node_subst_list(node, "commands") + self.add_commands("commands", cmds) + + self.set_work_dir() + self.set_install_root() + self.set_root_read_only(self.node_get_member(node, bool, + 'root-read-only', False)) + + +# Plugin entry point +def setup(): + return ScriptElement diff --git a/src/buildstream/plugins/elements/script.yaml b/src/buildstream/plugins/elements/script.yaml new file mode 100644 index 000000000..b388378da --- /dev/null +++ b/src/buildstream/plugins/elements/script.yaml @@ -0,0 +1,25 @@ +# Common script element variables +variables: + # Defines the directory commands will be run from. + cwd: / + +# Script element configuration +config: + + # Defines whether to run the sandbox with '/' read-only. + # It is recommended to set root as read-only wherever possible. + root-read-only: False + + # Defines where to stage elements which are direct or indirect dependencies. + # By default, all direct dependencies are staged to '/'. + # This is also commonly used to take one element as an environment + # containing the tools used to operate on the other element. + # layout: + # - element: foo-tools.bst + # destination: / + # - element: foo-system.bst + # destination: %{build-root} + + # List of commands to run in the sandbox. + commands: [] + diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py new file mode 100644 index 000000000..97517ca48 --- /dev/null +++ b/src/buildstream/plugins/elements/stack.py @@ -0,0 +1,66 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +stack - Symbolic Element for dependency grouping +================================================ +Stack elements are simply a symbolic element used for representing +a logical group of elements. +""" + +from buildstream import Element + + +# Element implementation for the 'stack' kind. +class StackElement(Element): + + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + # We do not add anything to the build, only our dependencies + # do, so our unique key is just a constant. + return 1 + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + + # Just create a dummy empty artifact, its existence is a statement + # that all this stack's dependencies are built. + vrootdir = sandbox.get_virtual_directory() + vrootdir.descend('output', create=True) + + # And we're done + return '/output' + + +# Plugin entry point +def setup(): + return StackElement diff --git a/src/buildstream/plugins/sources/__init__.py b/src/buildstream/plugins/sources/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/buildstream/plugins/sources/__init__.py diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py new file mode 100644 index 000000000..b9b15e268 --- /dev/null +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -0,0 +1,250 @@ +"""A base abstract class for source implementations which download a file""" + +import os +import urllib.request +import urllib.error +import contextlib +import shutil +import netrc + +from buildstream import Source, SourceError, Consistency +from buildstream import utils + + +class _NetrcFTPOpener(urllib.request.FTPHandler): + + def __init__(self, netrc_config): + self.netrc = netrc_config + + def _split(self, netloc): + userpass, hostport = urllib.parse.splituser(netloc) + host, port = urllib.parse.splitport(hostport) + if userpass: + user, passwd = urllib.parse.splitpasswd(userpass) + else: + user = None + passwd = None + return host, port, user, passwd + + def _unsplit(self, host, port, user, passwd): + if port: + host = '{}:{}'.format(host, port) + if user: + if passwd: + user = '{}:{}'.format(user, passwd) + host = '{}@{}'.format(user, host) + + return host + + def ftp_open(self, req): + host, port, user, passwd = self._split(req.host) + + if user is None and self.netrc: + entry = self.netrc.authenticators(host) + if entry: + user, _, passwd = entry + + req.host = self._unsplit(host, port, user, passwd) + + return super().ftp_open(req) + + +class _NetrcPasswordManager: + + def __init__(self, netrc_config): + self.netrc = netrc_config + + def add_password(self, realm, uri, user, passwd): + pass + + def find_user_password(self, realm, authuri): + if not self.netrc: + return None, None + parts = urllib.parse.urlsplit(authuri) + entry = self.netrc.authenticators(parts.hostname) + if not entry: + return None, None + else: + login, _, password = entry + return login, password + + +class DownloadableFileSource(Source): + # pylint: disable=attribute-defined-outside-init + + COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] + + __urlopener = None + + def configure(self, node): + self.original_url = self.node_get_member(node, str, 'url') + self.ref = self.node_get_member(node, str, 'ref', None) + self.url = self.translate_url(self.original_url) + self._warn_deprecated_etag(node) + + def preflight(self): + return + + def get_unique_key(self): + return [self.original_url, self.ref] + + def get_consistency(self): + if self.ref is None: + return Consistency.INCONSISTENT + + if os.path.isfile(self._get_mirror_file()): + return Consistency.CACHED + + else: + return Consistency.RESOLVED + + def load_ref(self, node): + self.ref = self.node_get_member(node, str, 'ref', None) + self._warn_deprecated_etag(node) + + def get_ref(self): + return self.ref + + def set_ref(self, ref, node): + node['ref'] = self.ref = ref + + def track(self): + # there is no 'track' field in the source to determine what/whether + # or not to update refs, because tracking a ref is always a conscious + # decision by the user. + with self.timed_activity("Tracking {}".format(self.url), + silent_nested=True): + new_ref = self._ensure_mirror() + + if self.ref and self.ref != new_ref: + detail = "When tracking, new ref differs from current ref:\n" \ + + " Tracked URL: {}\n".format(self.url) \ + + " Current ref: {}\n".format(self.ref) \ + + " New ref: {}\n".format(new_ref) + self.warn("Potential man-in-the-middle attack!", detail=detail) + + return new_ref + + def fetch(self): + + # Just a defensive check, it is impossible for the + # file to be already cached because Source.fetch() will + # not be called if the source is already Consistency.CACHED. + # + if os.path.isfile(self._get_mirror_file()): + return # pragma: nocover + + # Download the file, raise hell if the sha256sums don't match, + # and mirror the file otherwise. + with self.timed_activity("Fetching {}".format(self.url), silent_nested=True): + sha256 = self._ensure_mirror() + if sha256 != self.ref: + raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!" + .format(self.url, sha256, self.ref)) + + def _warn_deprecated_etag(self, node): + etag = self.node_get_member(node, str, 'etag', None) + if etag: + provenance = self.node_provenance(node, member_name='etag') + self.warn('{} "etag" is deprecated and ignored.'.format(provenance)) + + def _get_etag(self, ref): + etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref)) + if os.path.exists(etagfilename): + with open(etagfilename, 'r') as etagfile: + return etagfile.read() + + return None + + def _store_etag(self, ref, etag): + etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref)) + with utils.save_file_atomic(etagfilename) as etagfile: + etagfile.write(etag) + + def _ensure_mirror(self): + # Downloads from the url and caches it according to its sha256sum. + try: + with self.tempdir() as td: + default_name = os.path.basename(self.url) + request = urllib.request.Request(self.url) + request.add_header('Accept', '*/*') + + # We do not use etag in case what we have in cache is + # not matching ref in order to be able to recover from + # corrupted download. + if self.ref: + etag = self._get_etag(self.ref) + + # Do not re-download the file if the ETag matches. + if etag and self.get_consistency() == Consistency.CACHED: + request.add_header('If-None-Match', etag) + + opener = self.__get_urlopener() + with contextlib.closing(opener.open(request)) as response: + info = response.info() + + etag = info['ETag'] if 'ETag' in info else None + + filename = info.get_filename(default_name) + filename = os.path.basename(filename) + local_file = os.path.join(td, filename) + with open(local_file, 'wb') as dest: + shutil.copyfileobj(response, dest) + + # Make sure url-specific mirror dir exists. + if not os.path.isdir(self._get_mirror_dir()): + os.makedirs(self._get_mirror_dir()) + + # Store by sha256sum + sha256 = utils.sha256sum(local_file) + # Even if the file already exists, move the new file over. + # In case the old file was corrupted somehow. + os.rename(local_file, self._get_mirror_file(sha256)) + + if etag: + self._store_etag(sha256, etag) + return sha256 + + except urllib.error.HTTPError as e: + if e.code == 304: + # 304 Not Modified. + # Because we use etag only for matching ref, currently specified ref is what + # we would have downloaded. + return self.ref + raise SourceError("{}: Error mirroring {}: {}" + .format(self, self.url, e), temporary=True) from e + + except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError, ValueError) as e: + # Note that urllib.request.Request in the try block may throw a + # ValueError for unknown url types, so we handle it here. + raise SourceError("{}: Error mirroring {}: {}" + .format(self, self.url, e), temporary=True) from e + + def _get_mirror_dir(self): + return os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url)) + + def _get_mirror_file(self, sha=None): + return os.path.join(self._get_mirror_dir(), sha or self.ref) + + def __get_urlopener(self): + if not DownloadableFileSource.__urlopener: + try: + netrc_config = netrc.netrc() + except OSError: + # If the .netrc file was not found, FileNotFoundError will be + # raised, but OSError will be raised directly by the netrc package + # in the case that $HOME is not set. + # + # This will catch both cases. + # + DownloadableFileSource.__urlopener = urllib.request.build_opener() + except netrc.NetrcParseError as e: + self.warn('{}: While reading .netrc: {}'.format(self, e)) + return urllib.request.build_opener() + else: + netrc_pw_mgr = _NetrcPasswordManager(netrc_config) + http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr) + ftp_handler = _NetrcFTPOpener(netrc_config) + DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler) + return DownloadableFileSource.__urlopener diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py new file mode 100644 index 000000000..e59986da6 --- /dev/null +++ b/src/buildstream/plugins/sources/bzr.py @@ -0,0 +1,210 @@ +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jonathan Maw <jonathan.maw@codethink.co.uk> + +""" +bzr - stage files from a bazaar repository +========================================== + +**Host dependencies:** + + * bzr + +**Usage:** + +.. code:: yaml + + # Specify the bzr source kind + kind: bzr + + # Specify the bzr url. Bazaar URLs come in many forms, see + # `bzr help urlspec` for more information. Using an alias defined + # in your project configuration is encouraged. + url: https://launchpad.net/bzr + + # Specify the tracking branch. This is mandatory, as bzr cannot identify + # an individual revision outside its branch. bzr URLs that omit the branch + # name implicitly specify the trunk branch, but bst requires this to be + # explicit. + track: trunk + + # Specify the ref. This is a revision number. This is usually a decimal, + # but revisions on a branch are of the form + # <revision-branched-from>.<branch-number>.<revision-since-branching> + # e.g. 6622.1.6. + # The ref must be specified to build, and 'bst source track' will update the + # revision number to the one on the tip of the branch specified in 'track'. + ref: 6622 + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. +""" + +import os +import shutil +import fcntl +from contextlib import contextmanager + +from buildstream import Source, SourceError, Consistency +from buildstream import utils + + +class BzrSource(Source): + # pylint: disable=attribute-defined-outside-init + + def configure(self, node): + self.node_validate(node, ['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS]) + + self.original_url = self.node_get_member(node, str, 'url') + self.tracking = self.node_get_member(node, str, 'track') + self.ref = self.node_get_member(node, str, 'ref', None) + self.url = self.translate_url(self.original_url) + + def preflight(self): + # Check if bzr is installed, get the binary at the same time. + self.host_bzr = utils.get_host_tool('bzr') + + def get_unique_key(self): + return [self.original_url, self.tracking, self.ref] + + def get_consistency(self): + if self.ref is None or self.tracking is None: + return Consistency.INCONSISTENT + + # Lock for the _check_ref() + with self._locked(): + if self._check_ref(): + return Consistency.CACHED + else: + return Consistency.RESOLVED + + def load_ref(self, node): + self.ref = self.node_get_member(node, str, 'ref', None) + + def get_ref(self): + return self.ref + + def set_ref(self, ref, node): + node['ref'] = self.ref = ref + + def track(self): + with self.timed_activity("Tracking {}".format(self.url), + silent_nested=True), self._locked(): + self._ensure_mirror(skip_ref_check=True) + ret, out = self.check_output([self.host_bzr, "version-info", + "--custom", "--template={revno}", + self._get_branch_dir()], + fail="Failed to read the revision number at '{}'" + .format(self._get_branch_dir())) + if ret != 0: + raise SourceError("{}: Failed to get ref for tracking {}".format(self, self.tracking)) + + return out + + def fetch(self): + with self.timed_activity("Fetching {}".format(self.url), + silent_nested=True), self._locked(): + self._ensure_mirror() + + def stage(self, directory): + self.call([self.host_bzr, "checkout", "--lightweight", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir(), directory], + fail="Failed to checkout revision {} from branch {} to {}" + .format(self.ref, self._get_branch_dir(), directory)) + # Remove .bzr dir + shutil.rmtree(os.path.join(directory, ".bzr")) + + def init_workspace(self, directory): + url = os.path.join(self.url, self.tracking) + with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): + # Checkout from the cache + self.call([self.host_bzr, "branch", + "--use-existing-dir", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir(), directory], + fail="Failed to branch revision {} from branch {} to {}" + .format(self.ref, self._get_branch_dir(), directory)) + # Switch the parent branch to the source's origin + self.call([self.host_bzr, "switch", + "--directory={}".format(directory), url], + fail="Failed to switch workspace's parent branch to {}".format(url)) + + # _locked() + # + # This context manager ensures exclusive access to the + # bzr repository. + # + @contextmanager + def _locked(self): + lockdir = os.path.join(self.get_mirror_directory(), 'locks') + lockfile = os.path.join( + lockdir, + utils.url_directory_name(self.original_url) + '.lock' + ) + os.makedirs(lockdir, exist_ok=True) + with open(lockfile, 'w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock, fcntl.LOCK_UN) + + def _check_ref(self): + # If the mirror doesnt exist yet, then we dont have the ref + if not os.path.exists(self._get_branch_dir()): + return False + + return self.call([self.host_bzr, "revno", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir()]) == 0 + + def _get_branch_dir(self): + return os.path.join(self._get_mirror_dir(), self.tracking) + + def _get_mirror_dir(self): + return os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url)) + + def _ensure_mirror(self, skip_ref_check=False): + mirror_dir = self._get_mirror_dir() + bzr_metadata_dir = os.path.join(mirror_dir, ".bzr") + if not os.path.exists(bzr_metadata_dir): + self.call([self.host_bzr, "init-repo", "--no-trees", mirror_dir], + fail="Failed to initialize bzr repository") + + branch_dir = os.path.join(mirror_dir, self.tracking) + branch_url = self.url + "/" + self.tracking + if not os.path.exists(branch_dir): + # `bzr branch` the branch if it doesn't exist + # to get the upstream code + self.call([self.host_bzr, "branch", branch_url, branch_dir], + fail="Failed to branch from {} to {}".format(branch_url, branch_dir)) + + else: + # `bzr pull` the branch if it does exist + # to get any changes to the upstream code + self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url], + fail="Failed to pull new changes for {}".format(branch_dir)) + + if not skip_ref_check and not self._check_ref(): + raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), + reason="ref-not-mirrored") + + +def setup(): + return BzrSource diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py new file mode 100644 index 000000000..e45994951 --- /dev/null +++ b/src/buildstream/plugins/sources/deb.py @@ -0,0 +1,83 @@ +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Phillip Smyth <phillip.smyth@codethink.co.uk> +# Jonathan Maw <jonathan.maw@codethink.co.uk> +# Richard Maw <richard.maw@codethink.co.uk> + +""" +deb - stage files from .deb packages +==================================== + +**Host dependencies:** + + * arpy (python package) + +**Usage:** + +.. code:: yaml + + # Specify the deb source kind + kind: deb + + # Specify the deb url. Using an alias defined in your project + # configuration is encouraged. 'bst source track' will update the + # sha256sum in 'ref' to the downloaded file's sha256sum. + url: upstream:foo.deb + + # Specify the ref. It's a sha256sum of the file you download. + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b + + # Specify the basedir to return only the specified dir and its children + base-dir: '' + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. +""" + +import tarfile +from contextlib import contextmanager +import arpy # pylint: disable=import-error + +from .tar import TarSource + + +class DebSource(TarSource): + # pylint: disable=attribute-defined-outside-init + + def configure(self, node): + super().configure(node) + + self.base_dir = self.node_get_member(node, str, 'base-dir', None) + + def preflight(self): + return + + @contextmanager + def _get_tar(self): + with open(self._get_mirror_file(), 'rb') as deb_file: + arpy_archive = arpy.Archive(fileobj=deb_file) + arpy_archive.read_all_headers() + data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0] + # ArchiveFileData is not enough like a file object for tarfile to use. + # Monkey-patching a seekable method makes it close enough for TarFile to open. + data_tar_arpy.seekable = lambda *args: True + tar = tarfile.open(fileobj=data_tar_arpy, mode="r:*") + yield tar + + +def setup(): + return DebSource diff --git a/src/buildstream/plugins/sources/git.py b/src/buildstream/plugins/sources/git.py new file mode 100644 index 000000000..5e6834979 --- /dev/null +++ b/src/buildstream/plugins/sources/git.py @@ -0,0 +1,168 @@ +# +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +""" +git - stage files from a git repository +======================================= + +**Host dependencies:** + + * git + +.. attention:: + + Note that this plugin **will checkout git submodules by default**; even if + they are not specified in the `.bst` file. + +**Usage:** + +.. code:: yaml + + # Specify the git source kind + kind: git + + # Specify the repository url, using an alias defined + # in your project configuration is recommended. + url: upstream:foo.git + + # Optionally specify a symbolic tracking branch or tag, this + # will be used to update the 'ref' when refreshing the pipeline. + track: master + + # Optionally specify the ref format used for tracking. + # The default is 'sha1' for the raw commit hash. + # If you specify 'git-describe', the commit hash will be prefixed + # with the closest tag. + ref-format: sha1 + + # Specify the commit ref, this must be specified in order to + # checkout sources and build, but can be automatically updated + # if the 'track' attribute was specified. + ref: d63cbb6fdc0bbdadc4a1b92284826a6d63a7ebcd + + # Optionally specify whether submodules should be checked-out. + # If not set, this will default to 'True' + checkout-submodules: True + + # If your repository has submodules, explicitly specifying the + # url from which they are to be fetched allows you to easily + # rebuild the same sources from a different location. This is + # especially handy when used with project defined aliases which + # can be redefined at a later time. + # You may also explicitly specify whether to check out this + # submodule. If 'checkout' is set, it will override + # 'checkout-submodules' with the value set below. + submodules: + plugins/bar: + url: upstream:bar.git + checkout: True + plugins/baz: + url: upstream:baz.git + checkout: False + + # Enable tag tracking. + # + # This causes the `tags` metadata to be populated automatically + # as a result of tracking the git source. + # + # By default this is 'False'. + # + track-tags: True + + # If the list of tags below is set, then a lightweight dummy + # git repository will be staged along with the content at + # build time. + # + # This is useful for a growing number of modules which use + # `git describe` at build time in order to determine the version + # which will be encoded into the built software. + # + # The 'tags' below is considered as a part of the git source + # reference and will be stored in the 'project.refs' file if + # that has been selected as your project's ref-storage. + # + # Migration notes: + # + # If you are upgrading from BuildStream 1.2, which used to + # stage the entire repository by default, you will notice that + # some modules which use `git describe` are broken, and will + # need to enable this feature in order to fix them. + # + # If you need to enable this feature without changing the + # the specific commit that you are building, then we recommend + # the following migration steps for any git sources where + # `git describe` is required: + # + # o Enable `track-tags` feature + # o Set the `track` parameter to the desired commit sha which + # the current `ref` points to + # o Run `bst source track` for these elements, this will result in + # populating the `tags` portion of the refs without changing + # the refs + # o Restore the `track` parameter to the branches which you have + # previously been tracking afterwards. + # + tags: + - tag: lightweight-example + commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78 + annotated: false + - tag: annotated-example + commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a + annotated: true + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. + +**Configurable Warnings:** + +This plugin provides the following :ref:`configurable warnings <configurable_warnings>`: + +- ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never + added with `git submodule add`. + +- ``git:unlisted-submodule`` - A submodule is present in the git repository but was not specified in + the source configuration and was not disabled for checkout. + + .. note:: + + The ``git:unlisted-submodule`` warning is available since :ref:`format version 20 <project_format_version>` + +- ``git:invalid-submodule`` - A submodule is specified in the source configuration but does not exist + in the repository. + + .. note:: + + The ``git:invalid-submodule`` warning is available since :ref:`format version 20 <project_format_version>` + +This plugin also utilises the following configurable :class:`core warnings <buildstream.types.CoreWarnings>`: + +- :attr:`ref-not-in-track <buildstream.types.CoreWarnings.REF_NOT_IN_TRACK>` - The provided ref was not + found in the provided track in the element's git repository. +""" + +from buildstream import _GitSourceBase + + +class GitSource(_GitSourceBase): + pass + + +# Plugin entry point +def setup(): + return GitSource diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py new file mode 100644 index 000000000..50df85427 --- /dev/null +++ b/src/buildstream/plugins/sources/local.py @@ -0,0 +1,147 @@ +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# Tiago Gomes <tiago.gomes@codethink.co.uk> + +""" +local - stage local files and directories +========================================= + +**Usage:** + +.. code:: yaml + + # Specify the local source kind + kind: local + + # Specify the project relative path to a file or directory + path: files/somefile.txt + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. +""" + +import os +import stat +from buildstream import Source, Consistency +from buildstream import utils + + +class LocalSource(Source): + # pylint: disable=attribute-defined-outside-init + + def __init__(self, context, project, meta): + super().__init__(context, project, meta) + + # Cached unique key to avoid multiple file system traversal if the unique key is requested multiple times. + self.__unique_key = None + + def configure(self, node): + self.node_validate(node, ['path', *Source.COMMON_CONFIG_KEYS]) + self.path = self.node_get_project_path(node, 'path') + self.fullpath = os.path.join(self.get_project_directory(), self.path) + + def preflight(self): + pass + + def get_unique_key(self): + if self.__unique_key is None: + # Get a list of tuples of the the project relative paths and fullpaths + if os.path.isdir(self.fullpath): + filelist = utils.list_relative_paths(self.fullpath) + filelist = [(relpath, os.path.join(self.fullpath, relpath)) for relpath in filelist] + else: + filelist = [(self.path, self.fullpath)] + + # Return a list of (relative filename, sha256 digest) tuples, a sorted list + # has already been returned by list_relative_paths() + self.__unique_key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist] + return self.__unique_key + + def get_consistency(self): + return Consistency.CACHED + + # We dont have a ref, we're a local file... + def load_ref(self, node): + pass + + def get_ref(self): + return None # pragma: nocover + + def set_ref(self, ref, node): + pass # pragma: nocover + + def fetch(self): + # Nothing to do here for a local source + pass # pragma: nocover + + def stage(self, directory): + + # Dont use hardlinks to stage sources, they are not write protected + # in the sandbox. + with self.timed_activity("Staging local files at {}".format(self.path)): + + if os.path.isdir(self.fullpath): + files = list(utils.list_relative_paths(self.fullpath)) + utils.copy_files(self.fullpath, directory) + else: + destfile = os.path.join(directory, os.path.basename(self.path)) + files = [os.path.basename(self.path)] + utils.safe_copy(self.fullpath, destfile) + + for f in files: + # Non empty directories are not listed by list_relative_paths + dirs = f.split(os.sep) + for i in range(1, len(dirs)): + d = os.path.join(directory, *(dirs[:i])) + assert os.path.isdir(d) and not os.path.islink(d) + os.chmod(d, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + + path = os.path.join(directory, f) + if os.path.islink(path): + pass + elif os.path.isdir(path): + os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + else: + st = os.stat(path) + if st.st_mode & stat.S_IXUSR: + os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + else: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) + + def _get_local_path(self): + return self.fullpath + + +# Create a unique key for a file +def unique_key(filename): + + # Return some hard coded things for files which + # have no content to calculate a key for + if os.path.islink(filename): + # For a symbolic link, use the link target as its unique identifier + return os.readlink(filename) + elif os.path.isdir(filename): + return "0" + + return utils.sha256sum(filename) + + +# Plugin entry point +def setup(): + return LocalSource diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py new file mode 100644 index 000000000..e42868264 --- /dev/null +++ b/src/buildstream/plugins/sources/patch.py @@ -0,0 +1,101 @@ +# +# Copyright Bloomberg Finance LP +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Chandan Singh <csingh43@bloomberg.net> +# Tiago Gomes <tiago.gomes@codethink.co.uk> + +""" +patch - apply locally stored patches +==================================== + +**Host dependencies:** + + * patch + +**Usage:** + +.. code:: yaml + + # Specify the local source kind + kind: patch + + # Specify the project relative path to a patch file + path: files/somefile.diff + + # Optionally specify the strip level, defaults to 1 + strip-level: 1 + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. +""" + +import os +from buildstream import Source, SourceError, Consistency +from buildstream import utils + + +class PatchSource(Source): + # pylint: disable=attribute-defined-outside-init + + BST_REQUIRES_PREVIOUS_SOURCES_STAGE = True + + def configure(self, node): + self.path = self.node_get_project_path(node, 'path', + check_is_file=True) + self.strip_level = self.node_get_member(node, int, "strip-level", 1) + self.fullpath = os.path.join(self.get_project_directory(), self.path) + + def preflight(self): + # Check if patch is installed, get the binary at the same time + self.host_patch = utils.get_host_tool("patch") + + def get_unique_key(self): + return [self.path, utils.sha256sum(self.fullpath), self.strip_level] + + def get_consistency(self): + return Consistency.CACHED + + def load_ref(self, node): + pass + + def get_ref(self): + return None # pragma: nocover + + def set_ref(self, ref, node): + pass # pragma: nocover + + def fetch(self): + # Nothing to do here for a local source + pass # pragma: nocover + + def stage(self, directory): + with self.timed_activity("Applying local patch: {}".format(self.path)): + + # Bail out with a comprehensive message if the target directory is empty + if not os.listdir(directory): + raise SourceError("Nothing to patch in directory '{}'".format(directory), + reason="patch-no-files") + + strip_level_option = "-p{}".format(self.strip_level) + self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory], + fail="Failed to apply patch {}".format(self.path)) + + +# Plugin entry point +def setup(): + return PatchSource diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py new file mode 100644 index 000000000..9d6c40d74 --- /dev/null +++ b/src/buildstream/plugins/sources/pip.py @@ -0,0 +1,254 @@ +# +# Copyright 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Chandan Singh <csingh43@bloomberg.net> + +""" +pip - stage python packages using pip +===================================== + +**Host depndencies:** + + * ``pip`` python module + +This plugin will download source distributions for specified packages using +``pip`` but will not install them. It is expected that the elements using this +source will install the downloaded packages. + +Downloaded tarballs will be stored in a directory called ".bst_pip_downloads". + +**Usage:** + +.. code:: yaml + + # Specify the pip source kind + kind: pip + + # Optionally specify index url, defaults to PyPi + # This url is used to discover new versions of packages and download them + # Projects intending to mirror their sources to a permanent location should + # use an aliased url, and declare the alias in the project configuration + url: https://mypypi.example.com/simple + + # Optionally specify the path to requirements files + # Note that either 'requirements-files' or 'packages' must be defined + requirements-files: + - requirements.txt + + # Optionally specify a list of additional packages + # Note that either 'requirements-files' or 'packages' must be defined + packages: + - flake8 + + # Specify the ref. It is a list of strings of format + # "<package-name>==<version>", separated by "\\n". + # Usually this will be contents of a requirements.txt file where all + # package versions have been frozen. + ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0" + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. + +.. note:: + + The ``pip`` plugin is available since :ref:`format version 16 <project_format_version>` +""" + +import hashlib +import os +import re + +from buildstream import Consistency, Source, SourceError, utils + +_OUTPUT_DIRNAME = '.bst_pip_downloads' +_PYPI_INDEX_URL = 'https://pypi.org/simple/' + +# Used only for finding pip command +_PYTHON_VERSIONS = [ + 'python', # when running in a venv, we might not have the exact version + 'python2.7', + 'python3.0', + 'python3.1', + 'python3.2', + 'python3.3', + 'python3.4', + 'python3.5', + 'python3.6', + 'python3.7', +] + +# List of allowed extensions taken from +# https://docs.python.org/3/distutils/sourcedist.html. +# Names of source distribution archives must be of the form +# '%{package-name}-%{version}.%{extension}'. +_SDIST_RE = re.compile( + r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$', + re.IGNORECASE) + + +class PipSource(Source): + # pylint: disable=attribute-defined-outside-init + + # We need access to previous sources at track time to use requirements.txt + # but not at fetch time as self.ref should contain sufficient information + # for this plugin + BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True + + def configure(self, node): + self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] + + Source.COMMON_CONFIG_KEYS) + self.ref = self.node_get_member(node, str, 'ref', None) + self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL) + self.index_url = self.translate_url(self.original_url) + self.packages = self.node_get_member(node, list, 'packages', []) + self.requirements_files = self.node_get_member(node, list, 'requirements-files', []) + + if not (self.packages or self.requirements_files): + raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self)) + + def preflight(self): + # Try to find a pip version that supports download command + self.host_pip = None + for python in reversed(_PYTHON_VERSIONS): + try: + host_python = utils.get_host_tool(python) + rc = self.call([host_python, '-m', 'pip', 'download', '--help']) + if rc == 0: + self.host_pip = [host_python, '-m', 'pip'] + break + except utils.ProgramNotFoundError: + pass + + if self.host_pip is None: + raise SourceError("{}: Unable to find a suitable pip command".format(self)) + + def get_unique_key(self): + return [self.original_url, self.ref] + + def get_consistency(self): + if not self.ref: + return Consistency.INCONSISTENT + if os.path.exists(self._mirror) and os.listdir(self._mirror): + return Consistency.CACHED + return Consistency.RESOLVED + + def get_ref(self): + return self.ref + + def load_ref(self, node): + self.ref = self.node_get_member(node, str, 'ref', None) + + def set_ref(self, ref, node): + node['ref'] = self.ref = ref + + def track(self, previous_sources_dir): + # XXX pip does not offer any public API other than the CLI tool so it + # is not feasible to correctly parse the requirements file or to check + # which package versions pip is going to install. + # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program + # for details. + # As a result, we have to wastefully install the packages during track. + with self.tempdir() as tmpdir: + install_args = self.host_pip + ['download', + '--no-binary', ':all:', + '--index-url', self.index_url, + '--dest', tmpdir] + for requirement_file in self.requirements_files: + fpath = os.path.join(previous_sources_dir, requirement_file) + install_args += ['-r', fpath] + install_args += self.packages + + self.call(install_args, fail="Failed to install python packages") + reqs = self._parse_sdist_names(tmpdir) + + return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) + + def fetch(self): + with self.tempdir() as tmpdir: + packages = self.ref.strip().split('\n') + package_dir = os.path.join(tmpdir, 'packages') + os.makedirs(package_dir) + self.call([*self.host_pip, + 'download', + '--no-binary', ':all:', + '--index-url', self.index_url, + '--dest', package_dir, + *packages], + fail="Failed to install python packages: {}".format(packages)) + + # If the mirror directory already exists, assume that some other + # process has fetched the sources before us and ensure that we do + # not raise an error in that case. + try: + utils.move_atomic(package_dir, self._mirror) + except utils.DirectoryExistsError: + # Another process has beaten us and has fetched the sources + # before us. + pass + except OSError as e: + raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}" + .format(self, package_dir, self._mirror, e)) from e + + def stage(self, directory): + with self.timed_activity("Staging Python packages", silent_nested=True): + utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME)) + + # Directory where this source should stage its files + # + @property + def _mirror(self): + if not self.ref: + return None + return os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url), + hashlib.sha256(self.ref.encode()).hexdigest()) + + # Parse names of downloaded source distributions + # + # Args: + # basedir (str): Directory containing source distribution archives + # + # Returns: + # (list): List of (package_name, version) tuples in sorted order + # + def _parse_sdist_names(self, basedir): + reqs = [] + for f in os.listdir(basedir): + pkg = _match_package_name(f) + if pkg is not None: + reqs.append(pkg) + + return sorted(reqs) + + +# Extract the package name and version of a source distribution +# +# Args: +# filename (str): Filename of the source distribution +# +# Returns: +# (tuple): A tuple of (package_name, version) +# +def _match_package_name(filename): + pkg_match = _SDIST_RE.match(filename) + if pkg_match is None: + return None + return pkg_match.groups() + + +def setup(): + return PipSource diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py new file mode 100644 index 000000000..562a8f226 --- /dev/null +++ b/src/buildstream/plugins/sources/remote.py @@ -0,0 +1,93 @@ +# +# Copyright Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Ed Baunton <ebaunton1@bloomberg.net> + +""" +remote - stage files from remote urls +===================================== + +**Usage:** + +.. code:: yaml + + # Specify the remote source kind + kind: remote + + # Optionally specify a relative staging filename. + # If not specified, the basename of the url will be used. + # filename: customfilename + + # Optionally specify whether the downloaded file should be + # marked executable. + # executable: true + + # Specify the url. Using an alias defined in your project + # configuration is encouraged. 'bst source track' will update the + # sha256sum in 'ref' to the downloaded file's sha256sum. + url: upstream:foo + + # Specify the ref. It's a sha256sum of the file you download. + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. + +.. note:: + + The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>` +""" +import os +from buildstream import SourceError, utils +from ._downloadablefilesource import DownloadableFileSource + + +class RemoteSource(DownloadableFileSource): + # pylint: disable=attribute-defined-outside-init + + def configure(self, node): + super().configure(node) + + self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url)) + self.executable = self.node_get_member(node, bool, 'executable', False) + + if os.sep in self.filename: + raise SourceError('{}: filename parameter cannot contain directories'.format(self), + reason="filename-contains-directory") + self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable']) + + def get_unique_key(self): + return super().get_unique_key() + [self.filename, self.executable] + + def stage(self, directory): + # Same as in local plugin, don't use hardlinks to stage sources, they + # are not write protected in the sandbox. + dest = os.path.join(directory, self.filename) + with self.timed_activity("Staging remote file to {}".format(dest)): + + utils.safe_copy(self._get_mirror_file(), dest) + + # To prevent user's umask introducing variability here, explicitly set + # file modes. + if self.executable: + os.chmod(dest, 0o755) + else: + os.chmod(dest, 0o644) + + +def setup(): + return RemoteSource diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py new file mode 100644 index 000000000..31dc17497 --- /dev/null +++ b/src/buildstream/plugins/sources/tar.py @@ -0,0 +1,202 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jonathan Maw <jonathan.maw@codethink.co.uk> + +""" +tar - stage files from tar archives +=================================== + +**Host dependencies:** + + * lzip (for .tar.lz files) + +**Usage:** + +.. code:: yaml + + # Specify the tar source kind + kind: tar + + # Specify the tar url. Using an alias defined in your project + # configuration is encouraged. 'bst source track' will update the + # sha256sum in 'ref' to the downloaded file's sha256sum. + url: upstream:foo.tar + + # Specify the ref. It's a sha256sum of the file you download. + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b + + # Specify a glob pattern to indicate the base directory to extract + # from the tarball. The first matching directory will be used. + # + # Note that this is '*' by default since most standard release + # tarballs contain a self named subdirectory at the root which + # contains the files one normally wants to extract to build. + # + # To extract the root of the tarball directly, this can be set + # to an empty string. + base-dir: '*' + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. +""" + +import os +import tarfile +from contextlib import contextmanager +from tempfile import TemporaryFile + +from buildstream import SourceError +from buildstream import utils + +from ._downloadablefilesource import DownloadableFileSource + + +class TarSource(DownloadableFileSource): + # pylint: disable=attribute-defined-outside-init + + def configure(self, node): + super().configure(node) + + self.base_dir = self.node_get_member(node, str, 'base-dir', '*') or None + + self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + + def preflight(self): + self.host_lzip = None + if self.url.endswith('.lz'): + self.host_lzip = utils.get_host_tool('lzip') + + def get_unique_key(self): + return super().get_unique_key() + [self.base_dir] + + @contextmanager + def _run_lzip(self): + assert self.host_lzip + with TemporaryFile() as lzip_stdout: + with open(self._get_mirror_file(), 'r') as lzip_file: + self.call([self.host_lzip, '-d'], + stdin=lzip_file, + stdout=lzip_stdout) + + lzip_stdout.seek(0, 0) + yield lzip_stdout + + @contextmanager + def _get_tar(self): + if self.url.endswith('.lz'): + with self._run_lzip() as lzip_dec: + with tarfile.open(fileobj=lzip_dec, mode='r:') as tar: + yield tar + else: + with tarfile.open(self._get_mirror_file()) as tar: + yield tar + + def stage(self, directory): + try: + with self._get_tar() as tar: + base_dir = None + if self.base_dir: + base_dir = self._find_base_dir(tar, self.base_dir) + + if base_dir: + tar.extractall(path=directory, members=self._extract_members(tar, base_dir)) + else: + tar.extractall(path=directory) + + except (tarfile.TarError, OSError) as e: + raise SourceError("{}: Error staging source: {}".format(self, e)) from e + + # Override and translate which filenames to extract + def _extract_members(self, tar, base_dir): + if not base_dir.endswith(os.sep): + base_dir = base_dir + os.sep + + L = len(base_dir) + for member in tar.getmembers(): + + # First, ensure that a member never starts with `./` + if member.path.startswith('./'): + member.path = member.path[2:] + + # Now extract only the paths which match the normalized path + if member.path.startswith(base_dir): + + # If it's got a link name, give it the same treatment, we + # need the link targets to match up with what we are staging + # + # NOTE: Its possible this is not perfect, we may need to + # consider links which point outside of the chosen + # base directory. + # + if member.type == tarfile.LNKTYPE: + member.linkname = member.linkname[L:] + + member.path = member.path[L:] + yield member + + # We want to iterate over all paths of a tarball, but getmembers() + # is not enough because some tarballs simply do not contain the leading + # directory paths for the archived files. + def _list_tar_paths(self, tar): + + visited = set() + for member in tar.getmembers(): + + # Remove any possible leading './', offer more consistent behavior + # across tarballs encoded with or without a leading '.' + member_name = member.name.lstrip('./') + + if not member.isdir(): + + # Loop over the components of a path, for a path of a/b/c/d + # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding + # the final component + components = member_name.split('/') + for i in range(len(components) - 1): + dir_component = '/'.join([components[j] for j in range(i + 1)]) + if dir_component not in visited: + visited.add(dir_component) + try: + # Dont yield directory members which actually do + # exist in the archive + _ = tar.getmember(dir_component) + except KeyError: + if dir_component != '.': + yield dir_component + + continue + + # Avoid considering the '.' directory, if any is included in the archive + # this is to avoid the default 'base-dir: *' value behaving differently + # depending on whether the tarball was encoded with a leading '.' or not + elif member_name == '.': + continue + + yield member_name + + def _find_base_dir(self, tar, pattern): + paths = self._list_tar_paths(tar) + matches = sorted(list(utils.glob(paths, pattern))) + if not matches: + raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern)) + + return matches[0] + + +def setup(): + return TarSource diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py new file mode 100644 index 000000000..03efcef79 --- /dev/null +++ b/src/buildstream/plugins/sources/zip.py @@ -0,0 +1,181 @@ +# +# Copyright (C) 2017 Mathieu Bridon +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Mathieu Bridon <bochecha@daitauha.fr> + +""" +zip - stage files from zip archives +=================================== + +**Usage:** + +.. code:: yaml + + # Specify the zip source kind + kind: zip + + # Specify the zip url. Using an alias defined in your project + # configuration is encouraged. 'bst source track' will update the + # sha256sum in 'ref' to the downloaded file's sha256sum. + url: upstream:foo.zip + + # Specify the ref. It's a sha256sum of the file you download. + ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b + + # Specify a glob pattern to indicate the base directory to extract + # from the archive. The first matching directory will be used. + # + # Note that this is '*' by default since most standard release + # archives contain a self named subdirectory at the root which + # contains the files one normally wants to extract to build. + # + # To extract the root of the archive directly, this can be set + # to an empty string. + base-dir: '*' + +See :ref:`built-in functionality doumentation <core_source_builtins>` for +details on common configuration options for sources. + +.. attention:: + + File permissions are not preserved. All extracted directories have + permissions 0755 and all extracted files have permissions 0644. +""" + +import os +import zipfile +import stat + +from buildstream import SourceError +from buildstream import utils + +from ._downloadablefilesource import DownloadableFileSource + + +class ZipSource(DownloadableFileSource): + # pylint: disable=attribute-defined-outside-init + + def configure(self, node): + super().configure(node) + + self.base_dir = self.node_get_member(node, str, 'base-dir', '*') or None + + self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + + def get_unique_key(self): + return super().get_unique_key() + [self.base_dir] + + def stage(self, directory): + exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH) + noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + try: + with zipfile.ZipFile(self._get_mirror_file()) as archive: + base_dir = None + if self.base_dir: + base_dir = self._find_base_dir(archive, self.base_dir) + + if base_dir: + members = self._extract_members(archive, base_dir) + else: + members = archive.namelist() + + for member in members: + written = archive.extract(member, path=directory) + + # zipfile.extract might create missing directories + rel = os.path.relpath(written, start=directory) + assert not os.path.isabs(rel) + rel = os.path.dirname(rel) + while rel: + os.chmod(os.path.join(directory, rel), exec_rights) + rel = os.path.dirname(rel) + + if os.path.islink(written): + pass + elif os.path.isdir(written): + os.chmod(written, exec_rights) + else: + os.chmod(written, noexec_rights) + + except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e: + raise SourceError("{}: Error staging source: {}".format(self, e)) from e + + # Override and translate which filenames to extract + def _extract_members(self, archive, base_dir): + if not base_dir.endswith(os.sep): + base_dir = base_dir + os.sep + + L = len(base_dir) + for member in archive.infolist(): + if member.filename == base_dir: + continue + + if member.filename.startswith(base_dir): + member.filename = member.filename[L:] + yield member + + # We want to iterate over all paths of an archive, but namelist() + # is not enough because some archives simply do not contain the leading + # directory paths for the archived files. + def _list_archive_paths(self, archive): + + visited = {} + for member in archive.infolist(): + + # ZipInfo.is_dir() is only available in python >= 3.6, but all + # it does is check for a trailing '/' in the name + # + if not member.filename.endswith('/'): + + # Loop over the components of a path, for a path of a/b/c/d + # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding + # the final component + components = member.filename.split('/') + for i in range(len(components) - 1): + dir_component = '/'.join([components[j] for j in range(i + 1)]) + if dir_component not in visited: + visited[dir_component] = True + try: + # Dont yield directory members which actually do + # exist in the archive + _ = archive.getinfo(dir_component) + except KeyError: + if dir_component != '.': + yield dir_component + + continue + + # Avoid considering the '.' directory, if any is included in the archive + # this is to avoid the default 'base-dir: *' value behaving differently + # depending on whether the archive was encoded with a leading '.' or not + elif member.filename == '.' or member.filename == './': + continue + + yield member.filename + + def _find_base_dir(self, archive, pattern): + paths = self._list_archive_paths(archive) + matches = sorted(list(utils.glob(paths, pattern))) + if not matches: + raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern)) + + return matches[0] + + +def setup(): + return ZipSource |