summaryrefslogtreecommitdiff
path: root/src/buildstream/plugins
diff options
context:
space:
mode:
authorChandan Singh <csingh43@bloomberg.net>2019-04-24 22:53:19 +0100
committerChandan Singh <csingh43@bloomberg.net>2019-05-21 12:41:18 +0100
commit070d053e5cc47e572e9f9e647315082bd7a15c63 (patch)
tree7fb0fdff52f9b5f8a18ec8fe9c75b661f9e0839e /src/buildstream/plugins
parent6c59e7901a52be961c2a1b671cf2b30f90bc4d0a (diff)
downloadbuildstream-070d053e5cc47e572e9f9e647315082bd7a15c63.tar.gz
Move source from 'buildstream' to 'src/buildstream'
This was discussed in #1008. Fixes #1009.
Diffstat (limited to 'src/buildstream/plugins')
-rw-r--r--src/buildstream/plugins/elements/__init__.py0
-rw-r--r--src/buildstream/plugins/elements/autotools.py75
-rw-r--r--src/buildstream/plugins/elements/autotools.yaml129
-rw-r--r--src/buildstream/plugins/elements/cmake.py74
-rw-r--r--src/buildstream/plugins/elements/cmake.yaml72
-rw-r--r--src/buildstream/plugins/elements/compose.py194
-rw-r--r--src/buildstream/plugins/elements/compose.yaml34
-rw-r--r--src/buildstream/plugins/elements/distutils.py51
-rw-r--r--src/buildstream/plugins/elements/distutils.yaml49
-rw-r--r--src/buildstream/plugins/elements/filter.py256
-rw-r--r--src/buildstream/plugins/elements/filter.yaml29
-rw-r--r--src/buildstream/plugins/elements/import.py129
-rw-r--r--src/buildstream/plugins/elements/import.yaml14
-rw-r--r--src/buildstream/plugins/elements/junction.py229
-rw-r--r--src/buildstream/plugins/elements/make.py56
-rw-r--r--src/buildstream/plugins/elements/make.yaml42
-rw-r--r--src/buildstream/plugins/elements/makemaker.py51
-rw-r--r--src/buildstream/plugins/elements/makemaker.yaml48
-rw-r--r--src/buildstream/plugins/elements/manual.py51
-rw-r--r--src/buildstream/plugins/elements/manual.yaml22
-rw-r--r--src/buildstream/plugins/elements/meson.py71
-rw-r--r--src/buildstream/plugins/elements/meson.yaml79
-rw-r--r--src/buildstream/plugins/elements/modulebuild.py51
-rw-r--r--src/buildstream/plugins/elements/modulebuild.yaml48
-rw-r--r--src/buildstream/plugins/elements/pip.py51
-rw-r--r--src/buildstream/plugins/elements/pip.yaml36
-rw-r--r--src/buildstream/plugins/elements/qmake.py51
-rw-r--r--src/buildstream/plugins/elements/qmake.yaml50
-rw-r--r--src/buildstream/plugins/elements/script.py69
-rw-r--r--src/buildstream/plugins/elements/script.yaml25
-rw-r--r--src/buildstream/plugins/elements/stack.py66
-rw-r--r--src/buildstream/plugins/sources/__init__.py0
-rw-r--r--src/buildstream/plugins/sources/_downloadablefilesource.py250
-rw-r--r--src/buildstream/plugins/sources/bzr.py210
-rw-r--r--src/buildstream/plugins/sources/deb.py83
-rw-r--r--src/buildstream/plugins/sources/git.py168
-rw-r--r--src/buildstream/plugins/sources/local.py147
-rw-r--r--src/buildstream/plugins/sources/patch.py101
-rw-r--r--src/buildstream/plugins/sources/pip.py254
-rw-r--r--src/buildstream/plugins/sources/remote.py93
-rw-r--r--src/buildstream/plugins/sources/tar.py202
-rw-r--r--src/buildstream/plugins/sources/zip.py181
42 files changed, 3891 insertions, 0 deletions
diff --git a/src/buildstream/plugins/elements/__init__.py b/src/buildstream/plugins/elements/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/buildstream/plugins/elements/__init__.py
diff --git a/src/buildstream/plugins/elements/autotools.py b/src/buildstream/plugins/elements/autotools.py
new file mode 100644
index 000000000..7a05336b7
--- /dev/null
+++ b/src/buildstream/plugins/elements/autotools.py
@@ -0,0 +1,75 @@
+#
+# Copyright (C) 2016, 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+autotools - Autotools build element
+===================================
+This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
+using Autotools build scripts (also known as the `GNU Build System
+<https://en.wikipedia.org/wiki/GNU_Build_System>`_).
+
+You will often want to pass additional arguments to ``configure``. This should
+be done on a per-element basis by setting the ``conf-local`` variable. Here is
+an example:
+
+.. code:: yaml
+
+ variables:
+ conf-local: |
+ --disable-foo --enable-bar
+
+If you want to pass extra options to ``configure`` for every element in your
+project, set the ``conf-global`` variable in your project.conf file. Here is
+an example of that:
+
+.. code:: yaml
+
+ elements:
+ autotools:
+ variables:
+ conf-global: |
+ --disable-gtk-doc --disable-static
+
+Here is the default configuration for the ``autotools`` element in full:
+
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/autotools.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'autotools' kind.
+class AutotoolsElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return AutotoolsElement
diff --git a/src/buildstream/plugins/elements/autotools.yaml b/src/buildstream/plugins/elements/autotools.yaml
new file mode 100644
index 000000000..85f7393e7
--- /dev/null
+++ b/src/buildstream/plugins/elements/autotools.yaml
@@ -0,0 +1,129 @@
+# Autotools default configurations
+
+variables:
+
+ autogen: |
+ export NOCONFIGURE=1;
+
+ if [ -x %{conf-cmd} ]; then true;
+ elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen;
+ elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh;
+ elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap;
+ elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh;
+ else autoreconf -ivf %{conf-root};
+ fi
+
+ # Project-wide extra arguments to be passed to `configure`
+ conf-global: ''
+
+ # Element-specific extra arguments to be passed to `configure`.
+ conf-local: ''
+
+ # For backwards compatibility only, do not use.
+ conf-extra: ''
+
+ conf-cmd: "%{conf-root}/configure"
+
+ conf-args: |
+
+ --prefix=%{prefix} \
+ --exec-prefix=%{exec_prefix} \
+ --bindir=%{bindir} \
+ --sbindir=%{sbindir} \
+ --sysconfdir=%{sysconfdir} \
+ --datadir=%{datadir} \
+ --includedir=%{includedir} \
+ --libdir=%{libdir} \
+ --libexecdir=%{libexecdir} \
+ --localstatedir=%{localstatedir} \
+ --sharedstatedir=%{sharedstatedir} \
+ --mandir=%{mandir} \
+ --infodir=%{infodir} %{conf-extra} %{conf-global} %{conf-local}
+
+ configure: |
+
+ %{conf-cmd} %{conf-args}
+
+ make: make
+ make-install: make -j1 DESTDIR="%{install-root}" install
+
+ # Set this if the sources cannot handle parallelization.
+ #
+ # notparallel: True
+
+
+ # Automatically remove libtool archive files
+ #
+ # Set remove-libtool-modules to "true" to remove .la files for
+ # modules intended to be opened with lt_dlopen()
+ #
+ # Set remove-libtool-libraries to "true" to remove .la files for
+ # libraries
+ #
+ # Value must be "true" or "false"
+ remove-libtool-modules: "false"
+ remove-libtool-libraries: "false"
+
+ delete-libtool-archives: |
+ if %{remove-libtool-modules} || %{remove-libtool-libraries}; then
+ find "%{install-root}" -name "*.la" -print0 | while read -d '' -r file; do
+ if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then
+ if %{remove-libtool-modules}; then
+ echo "Removing ${file}."
+ rm "${file}"
+ else
+ echo "Not removing ${file}."
+ fi
+ else
+ if %{remove-libtool-libraries}; then
+ echo "Removing ${file}."
+ rm "${file}"
+ else
+ echo "Not removing ${file}."
+ fi
+ fi
+ done
+ fi
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands:
+ - |
+ %{autogen}
+ - |
+ %{configure}
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{make}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{make-install}
+ - |
+ %{delete-libtool-archives}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+ MAKEFLAGS: -j%{max-jobs}
+ V: 1
+
+# And dont consider MAKEFLAGS or V as something which may
+# affect build output.
+environment-nocache:
+- MAKEFLAGS
+- V
diff --git a/src/buildstream/plugins/elements/cmake.py b/src/buildstream/plugins/elements/cmake.py
new file mode 100644
index 000000000..74da04899
--- /dev/null
+++ b/src/buildstream/plugins/elements/cmake.py
@@ -0,0 +1,74 @@
+#
+# Copyright (C) 2016, 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+cmake - CMake build element
+===========================
+This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
+using the `CMake <https://cmake.org/>`_ build system.
+
+You will often want to pass additional arguments to the ``cmake`` program for
+specific configuration options. This should be done on a per-element basis by
+setting the ``cmake-local`` variable. Here is an example:
+
+.. code:: yaml
+
+ variables:
+ cmake-local: |
+ -DCMAKE_BUILD_TYPE=Debug
+
+If you want to pass extra options to ``cmake`` for every element in your
+project, set the ``cmake-global`` variable in your project.conf file. Here is
+an example of that:
+
+.. code:: yaml
+
+ elements:
+ cmake:
+ variables:
+ cmake-global: |
+ -DCMAKE_BUILD_TYPE=Release
+
+Here is the default configuration for the ``cmake`` element in full:
+
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/cmake.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'cmake' kind.
+class CMakeElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return CMakeElement
diff --git a/src/buildstream/plugins/elements/cmake.yaml b/src/buildstream/plugins/elements/cmake.yaml
new file mode 100644
index 000000000..ba20d7ce6
--- /dev/null
+++ b/src/buildstream/plugins/elements/cmake.yaml
@@ -0,0 +1,72 @@
+# CMake default configuration
+
+variables:
+
+ build-dir: _builddir
+
+ # Project-wide extra arguments to be passed to `cmake`
+ cmake-global: ''
+
+ # Element-specific extra arguments to be passed to `cmake`.
+ cmake-local: ''
+
+ # For backwards compatibility only, do not use.
+ cmake-extra: ''
+
+ # The cmake generator to use
+ generator: Unix Makefiles
+
+ cmake-args: |
+
+ -DCMAKE_INSTALL_PREFIX:PATH="%{prefix}" \
+ -DCMAKE_INSTALL_LIBDIR:PATH="%{lib}" %{cmake-extra} %{cmake-global} %{cmake-local}
+
+ cmake: |
+
+ cmake -B%{build-dir} -H"%{conf-root}" -G"%{generator}" %{cmake-args}
+
+ make: cmake --build %{build-dir} -- ${JOBS}
+ make-install: env DESTDIR="%{install-root}" cmake --build %{build-dir} --target install
+
+ # Set this if the sources cannot handle parallelization.
+ #
+ # notparallel: True
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands:
+ - |
+ %{cmake}
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{make}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{make-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+ JOBS: -j%{max-jobs}
+ V: 1
+
+# And dont consider JOBS or V as something which may
+# affect build output.
+environment-nocache:
+- JOBS
+- V
diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py
new file mode 100644
index 000000000..b672cde0c
--- /dev/null
+++ b/src/buildstream/plugins/elements/compose.py
@@ -0,0 +1,194 @@
+#
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+compose - Compose the output of multiple elements
+=================================================
+This element creates a selective composition of its dependencies.
+
+This is normally used at near the end of a pipeline to prepare
+something for later deployment.
+
+Since this element's output includes its dependencies, it may only
+depend on elements as `build` type dependencies.
+
+The default configuration and possible options are as such:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/compose.yaml
+ :language: yaml
+"""
+
+import os
+from buildstream import Element, Scope
+
+
+# Element implementation for the 'compose' kind.
+class ComposeElement(Element):
+ # pylint: disable=attribute-defined-outside-init
+
+ # The compose element's output is its dependencies, so
+ # we must rebuild if the dependencies change even when
+ # not in strict build plans.
+ #
+ BST_STRICT_REBUILD = True
+
+ # Compose artifacts must never have indirect dependencies,
+ # so runtime dependencies are forbidden.
+ BST_FORBID_RDEPENDS = True
+
+ # This element ignores sources, so we should forbid them from being
+ # added, to reduce the potential for confusion
+ BST_FORBID_SOURCES = True
+
+ # This plugin has been modified to avoid the use of Sandbox.get_directory
+ BST_VIRTUAL_DIRECTORY = True
+
+ def configure(self, node):
+ self.node_validate(node, [
+ 'integrate', 'include', 'exclude', 'include-orphans'
+ ])
+
+ # We name this variable 'integration' only to avoid
+ # collision with the Element.integrate() method.
+ self.integration = self.node_get_member(node, bool, 'integrate')
+ self.include = self.node_get_member(node, list, 'include')
+ self.exclude = self.node_get_member(node, list, 'exclude')
+ self.include_orphans = self.node_get_member(node, bool, 'include-orphans')
+
+ def preflight(self):
+ pass
+
+ def get_unique_key(self):
+ key = {'integrate': self.integration,
+ 'include': sorted(self.include),
+ 'orphans': self.include_orphans}
+
+ if self.exclude:
+ key['exclude'] = sorted(self.exclude)
+
+ return key
+
+ def configure_sandbox(self, sandbox):
+ pass
+
+ def stage(self, sandbox):
+ pass
+
+ def assemble(self, sandbox):
+
+ require_split = self.include or self.exclude or not self.include_orphans
+
+ # Stage deps in the sandbox root
+ with self.timed_activity("Staging dependencies", silent_nested=True):
+ self.stage_dependency_artifacts(sandbox, Scope.BUILD)
+
+ manifest = set()
+ if require_split:
+ with self.timed_activity("Computing split", silent_nested=True):
+ for dep in self.dependencies(Scope.BUILD):
+ files = dep.compute_manifest(include=self.include,
+ exclude=self.exclude,
+ orphans=self.include_orphans)
+ manifest.update(files)
+
+ # Make a snapshot of all the files.
+ vbasedir = sandbox.get_virtual_directory()
+ modified_files = set()
+ removed_files = set()
+ added_files = set()
+
+ # Run any integration commands provided by the dependencies
+ # once they are all staged and ready
+ if self.integration:
+ with self.timed_activity("Integrating sandbox"):
+ if require_split:
+
+ # Make a snapshot of all the files before integration-commands are run.
+ snapshot = set(vbasedir.list_relative_paths())
+ vbasedir.mark_unmodified()
+
+ with sandbox.batch(0):
+ for dep in self.dependencies(Scope.BUILD):
+ dep.integrate(sandbox)
+
+ if require_split:
+ # Calculate added, modified and removed files
+ post_integration_snapshot = vbasedir.list_relative_paths()
+ modified_files = set(vbasedir.list_modified_paths())
+ basedir_contents = set(post_integration_snapshot)
+ for path in manifest:
+ if path in snapshot and path not in basedir_contents:
+ removed_files.add(path)
+
+ for path in basedir_contents:
+ if path not in snapshot:
+ added_files.add(path)
+ self.info("Integration modified {}, added {} and removed {} files"
+ .format(len(modified_files), len(added_files), len(removed_files)))
+
+ # The remainder of this is expensive, make an early exit if
+ # we're not being selective about what is to be included.
+ if not require_split:
+ return '/'
+
+ # Do we want to force include files which were modified by
+ # the integration commands, even if they were not added ?
+ #
+ manifest.update(added_files)
+ manifest.difference_update(removed_files)
+
+ # XXX We should be moving things outside of the build sandbox
+ # instead of into a subdir. The element assemble() method should
+ # support this in some way.
+ #
+ installdir = vbasedir.descend('buildstream', 'install', create=True)
+
+ # We already saved the manifest for created files in the integration phase,
+ # now collect the rest of the manifest.
+ #
+
+ lines = []
+ if self.include:
+ lines.append("Including files from domains: " + ", ".join(self.include))
+ else:
+ lines.append("Including files from all domains")
+
+ if self.exclude:
+ lines.append("Excluding files from domains: " + ", ".join(self.exclude))
+
+ if self.include_orphans:
+ lines.append("Including orphaned files")
+ else:
+ lines.append("Excluding orphaned files")
+
+ detail = "\n".join(lines)
+
+ def import_filter(path):
+ return path in manifest
+
+ with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
+ self.info("Composing {} files".format(len(manifest)))
+ installdir.import_files(vbasedir, filter_callback=import_filter, can_link=True)
+
+ # And we're done
+ return os.path.join(os.sep, 'buildstream', 'install')
+
+
+# Plugin entry point
+def setup():
+ return ComposeElement
diff --git a/src/buildstream/plugins/elements/compose.yaml b/src/buildstream/plugins/elements/compose.yaml
new file mode 100644
index 000000000..fd2eb9358
--- /dev/null
+++ b/src/buildstream/plugins/elements/compose.yaml
@@ -0,0 +1,34 @@
+
+# Compose element configuration
+config:
+
+ # Whether to run the integration commands for the
+ # staged dependencies.
+ #
+ integrate: True
+
+ # A list of domains to include from each artifact, as
+ # they were defined in the element's 'split-rules'.
+ #
+ # Since domains can be added, it is not an error to
+ # specify domains which may not exist for all of the
+ # elements in this composition.
+ #
+ # The default empty list indicates that all domains
+ # from each dependency should be included.
+ #
+ include: []
+
+ # A list of domains to exclude from each artifact, as
+ # they were defined in the element's 'split-rules'.
+ #
+ # In the case that a file is spoken for by a domain
+ # in the 'include' list and another in the 'exclude'
+ # list, then the file will be excluded.
+ exclude: []
+
+ # Whether to include orphan files which are not
+ # included by any of the 'split-rules' present on
+ # a given element.
+ #
+ include-orphans: True
diff --git a/src/buildstream/plugins/elements/distutils.py b/src/buildstream/plugins/elements/distutils.py
new file mode 100644
index 000000000..4b2c1e2f4
--- /dev/null
+++ b/src/buildstream/plugins/elements/distutils.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+distutils - Python distutils element
+====================================
+A :mod:`BuildElement <buildstream.buildelement>` implementation for using
+python distutils
+
+The distutils default configuration:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/distutils.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the python 'distutils' kind.
+class DistutilsElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return DistutilsElement
diff --git a/src/buildstream/plugins/elements/distutils.yaml b/src/buildstream/plugins/elements/distutils.yaml
new file mode 100644
index 000000000..cec7da6e9
--- /dev/null
+++ b/src/buildstream/plugins/elements/distutils.yaml
@@ -0,0 +1,49 @@
+# Default python distutils configuration
+
+variables:
+
+ # When building for python2 distutils, simply
+ # override this in the element declaration
+ python: python3
+
+ python-build: |
+
+ %{python} %{conf-root}/setup.py build
+
+ install-args: |
+
+ --prefix "%{prefix}" \
+ --root "%{install-root}"
+
+ python-install: |
+
+ %{python} %{conf-root}/setup.py install %{install-args}
+
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands: []
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{python-build}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{python-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+ - |
+ %{fix-pyc-timestamps}
diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py
new file mode 100644
index 000000000..45847e685
--- /dev/null
+++ b/src/buildstream/plugins/elements/filter.py
@@ -0,0 +1,256 @@
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jonathan Maw <jonathan.maw@codethink.co.uk>
+
+"""
+filter - Extract a subset of files from another element
+=======================================================
+Filter another element by producing an output that is a subset of
+the parent element's output. Subsets are defined by the parent element's
+:ref:`split rules <public_split_rules>`.
+
+Overview
+--------
+A filter element must have exactly one *build* dependency, where said
+dependency is the 'parent' element which we would like to filter.
+Runtime dependencies may also be specified, which can be useful to propagate
+forward from this filter element onto its reverse dependencies.
+See :ref:`Dependencies <format_dependencies>` to see how we specify dependencies.
+
+When workspaces are opened, closed or reset on a filter element, or this
+element is tracked, the filter element will transparently pass on the command
+to its parent element (the sole build-dependency).
+
+Example
+-------
+Consider a simple import element, ``import.bst`` which imports the local files
+'foo', 'bar' and 'baz' (each stored in ``files/``, relative to the project's root):
+
+.. code:: yaml
+
+ kind: import
+
+ # Specify sources to import
+ sources:
+ - kind: local
+ path: files
+
+ # Specify public domain data, visible to other elements
+ public:
+ bst:
+ split-rules:
+ foo:
+ - /foo
+ bar:
+ - /bar
+
+.. note::
+
+ We can make an element's metadata visible to all reverse dependencies by making use
+ of the ``public:`` field. See the :ref:`public data documentation <format_public>`
+ for more information.
+
+In this example, ``import.bst`` will serve as the 'parent' of the filter element, thus
+its output will be filtered. It is important to understand that the artifact of the
+above element will contain the files: 'foo', 'bar' and 'baz'.
+
+Now, to produce an element whose artifact contains the file 'foo', and exlusively 'foo',
+we can define the following filter, ``filter-foo.bst``:
+
+.. code:: yaml
+
+ kind: filter
+
+ # Declare the sole build-dependency of the filter element
+ depends:
+ - filename: import.bst
+ type: build
+
+ # Declare a list of domains to include in the filter's artifact
+ config:
+ include:
+ - foo
+
+.. note::
+
+ We can also specify build-dependencies with a 'build-depends' field which has been
+ available since :ref:`format version 14 <project_format_version>`. See the
+ :ref:`Build-Depends documentation <format_build_depends>` for more detail.
+
+It should be noted that an 'empty' ``include:`` list would, by default, include all
+split-rules specified in the parent element, which, in this example, would be the
+files 'foo' and 'bar' (the file 'baz' was not covered by any split rules).
+
+Equally, we can use the ``exclude:`` statement to create the same artifact (which
+only contains the file 'foo') by declaring the following element, ``exclude-bar.bst``:
+
+.. code:: yaml
+
+ kind: filter
+
+ # Declare the sole build-dependency of the filter element
+ depends:
+ - filename: import.bst
+ type: build
+
+ # Declare a list of domains to exclude in the filter's artifact
+ config:
+ exclude:
+ - bar
+
+In addition to the ``include:`` and ``exclude:`` fields, there exists an ``include-orphans:``
+(Boolean) field, which defaults to ``False``. This will determine whether to include files
+which are not present in the 'split-rules'. For example, if we wanted to filter out all files
+which are not included as split rules we can define the following element, ``filter-misc.bst``:
+
+.. code:: yaml
+
+ kind: filter
+
+ # Declare the sole build-dependency of the filter element
+ depends:
+ - filename: import.bst
+ type: build
+
+ # Filter out all files which are not declared as split rules
+ config:
+ exclude:
+ - foo
+ - bar
+ include-orphans: True
+
+The artifact of ``filter-misc.bst`` will only contain the file 'baz'.
+
+Below is more information regarding the the default configurations and possible options
+of the filter element:
+
+.. literalinclude:: ../../../src/buildstream/plugins/elements/filter.yaml
+ :language: yaml
+"""
+
+from buildstream import Element, ElementError, Scope
+
+
+class FilterElement(Element):
+ # pylint: disable=attribute-defined-outside-init
+
+ BST_ARTIFACT_VERSION = 1
+
+ # The filter element's output is its dependencies, so
+ # we must rebuild if the dependencies change even when
+ # not in strict build plans.
+ BST_STRICT_REBUILD = True
+
+ # This element ignores sources, so we should forbid them from being
+ # added, to reduce the potential for confusion
+ BST_FORBID_SOURCES = True
+
+ # This plugin has been modified to avoid the use of Sandbox.get_directory
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Filter elements do not run any commands
+ BST_RUN_COMMANDS = False
+
+ def configure(self, node):
+ self.node_validate(node, [
+ 'include', 'exclude', 'include-orphans'
+ ])
+
+ self.include = self.node_get_member(node, list, 'include')
+ self.exclude = self.node_get_member(node, list, 'exclude')
+ self.include_orphans = self.node_get_member(node, bool, 'include-orphans')
+ self.include_provenance = self.node_provenance(node, member_name='include')
+ self.exclude_provenance = self.node_provenance(node, member_name='exclude')
+
+ def preflight(self):
+ # Exactly one build-depend is permitted
+ build_deps = list(self.dependencies(Scope.BUILD, recurse=False))
+ if len(build_deps) != 1:
+ detail = "Full list of build-depends:\n"
+ deps_list = " \n".join([x.name for x in build_deps])
+ detail += deps_list
+ raise ElementError("{}: {} element must have exactly 1 build-dependency, actually have {}"
+ .format(self, type(self).__name__, len(build_deps)),
+ detail=detail, reason="filter-bdepend-wrong-count")
+
+ # That build-depend must not also be a runtime-depend
+ runtime_deps = list(self.dependencies(Scope.RUN, recurse=False))
+ if build_deps[0] in runtime_deps:
+ detail = "Full list of runtime depends:\n"
+ deps_list = " \n".join([x.name for x in runtime_deps])
+ detail += deps_list
+ raise ElementError("{}: {} element's build dependency must not also be a runtime dependency"
+ .format(self, type(self).__name__),
+ detail=detail, reason="filter-bdepend-also-rdepend")
+
+ def get_unique_key(self):
+ key = {
+ 'include': sorted(self.include),
+ 'exclude': sorted(self.exclude),
+ 'orphans': self.include_orphans,
+ }
+ return key
+
+ def configure_sandbox(self, sandbox):
+ pass
+
+ def stage(self, sandbox):
+ pass
+
+ def assemble(self, sandbox):
+ with self.timed_activity("Staging artifact", silent_nested=True):
+ for dep in self.dependencies(Scope.BUILD, recurse=False):
+ # Check that all the included/excluded domains exist
+ pub_data = dep.get_public_data('bst')
+ split_rules = self.node_get_member(pub_data, dict, 'split-rules', {})
+ unfound_includes = []
+ for domain in self.include:
+ if domain not in split_rules:
+ unfound_includes.append(domain)
+ unfound_excludes = []
+ for domain in self.exclude:
+ if domain not in split_rules:
+ unfound_excludes.append(domain)
+
+ detail = []
+ if unfound_includes:
+ detail.append("Unknown domains were used in {}".format(self.include_provenance))
+ detail.extend([' - {}'.format(domain) for domain in unfound_includes])
+
+ if unfound_excludes:
+ detail.append("Unknown domains were used in {}".format(self.exclude_provenance))
+ detail.extend([' - {}'.format(domain) for domain in unfound_excludes])
+
+ if detail:
+ detail = '\n'.join(detail)
+ raise ElementError("Unknown domains declared.", detail=detail)
+
+ dep.stage_artifact(sandbox, include=self.include,
+ exclude=self.exclude, orphans=self.include_orphans)
+ return ""
+
+ def _get_source_element(self):
+ # Filter elements act as proxies for their sole build-dependency
+ build_deps = list(self.dependencies(Scope.BUILD, recurse=False))
+ assert len(build_deps) == 1
+ output_elm = build_deps[0]._get_source_element()
+ return output_elm
+
+
+def setup():
+ return FilterElement
diff --git a/src/buildstream/plugins/elements/filter.yaml b/src/buildstream/plugins/elements/filter.yaml
new file mode 100644
index 000000000..9c2bf69f4
--- /dev/null
+++ b/src/buildstream/plugins/elements/filter.yaml
@@ -0,0 +1,29 @@
+
+# Filter element configuration
+config:
+
+ # A list of domains to include in each artifact, as
+ # they were defined as public data in the parent
+ # element's 'split-rules'.
+ #
+ # If a domain is specified that does not exist, the
+ # filter element will fail to build.
+ #
+ # The default empty list indicates that all domains
+ # of the parent's artifact should be included.
+ #
+ include: []
+
+ # A list of domains to exclude from each artifact, as
+ # they were defined in the parent element's 'split-rules'.
+ #
+ # In the case that a file is spoken for by a domain
+ # in the 'include' list and another in the 'exclude'
+ # list, then the file will be excluded.
+ exclude: []
+
+ # Whether to include orphan files which are not
+ # included by any of the 'split-rules' present in
+ # the parent element.
+ #
+ include-orphans: False
diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py
new file mode 100644
index 000000000..61e353dbc
--- /dev/null
+++ b/src/buildstream/plugins/elements/import.py
@@ -0,0 +1,129 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+import - Import sources directly
+================================
+Import elements produce artifacts directly from its sources
+without any kind of processing. These are typically used to
+import an SDK to build on top of or to overlay your build with
+some configuration data.
+
+The empty configuration is as such:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/import.yaml
+ :language: yaml
+"""
+
+import os
+from buildstream import Element, ElementError
+
+
+# Element implementation for the 'import' kind.
+class ImportElement(Element):
+ # pylint: disable=attribute-defined-outside-init
+
+ # This plugin has been modified to avoid the use of Sandbox.get_directory
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Import elements do not run any commands
+ BST_RUN_COMMANDS = False
+
+ def configure(self, node):
+ self.node_validate(node, [
+ 'source', 'target'
+ ])
+
+ self.source = self.node_subst_member(node, 'source')
+ self.target = self.node_subst_member(node, 'target')
+
+ def preflight(self):
+ # Assert that we have at least one source to fetch.
+
+ sources = list(self.sources())
+ if not sources:
+ raise ElementError("{}: An import element must have at least one source.".format(self))
+
+ def get_unique_key(self):
+ return {
+ 'source': self.source,
+ 'target': self.target
+ }
+
+ def configure_sandbox(self, sandbox):
+ pass
+
+ def stage(self, sandbox):
+ pass
+
+ def assemble(self, sandbox):
+
+ # Stage sources into the input directory
+ # Do not mount workspaces as the files are copied from outside the sandbox
+ self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
+
+ rootdir = sandbox.get_virtual_directory()
+ inputdir = rootdir.descend('input')
+ outputdir = rootdir.descend('output', create=True)
+
+ # The directory to grab
+ inputdir = inputdir.descend(*self.source.strip(os.sep).split(os.sep))
+
+ # The output target directory
+ outputdir = outputdir.descend(*self.target.strip(os.sep).split(os.sep), create=True)
+
+ if inputdir.is_empty():
+ raise ElementError("{}: No files were found inside directory '{}'"
+ .format(self, self.source))
+
+ # Move it over
+ outputdir.import_files(inputdir)
+
+ # And we're done
+ return '/output'
+
+ def generate_script(self):
+ build_root = self.get_variable('build-root')
+ install_root = self.get_variable('install-root')
+ commands = []
+
+ # The directory to grab
+ inputdir = os.path.join(build_root, self.normal_name, self.source.lstrip(os.sep))
+ inputdir = inputdir.rstrip(os.sep)
+
+ # The output target directory
+ outputdir = os.path.join(install_root, self.target.lstrip(os.sep))
+ outputdir = outputdir.rstrip(os.sep)
+
+ # Ensure target directory parent exists but target directory doesn't
+ commands.append("mkdir -p {}".format(os.path.dirname(outputdir)))
+ commands.append("[ ! -e {outputdir} ] || rmdir {outputdir}".format(outputdir=outputdir))
+
+ # Move it over
+ commands.append("mv {} {}".format(inputdir, outputdir))
+
+ script = ""
+ for cmd in commands:
+ script += "(set -ex; {}\n) || exit 1\n".format(cmd)
+
+ return script
+
+
+# Plugin entry point
+def setup():
+ return ImportElement
diff --git a/src/buildstream/plugins/elements/import.yaml b/src/buildstream/plugins/elements/import.yaml
new file mode 100644
index 000000000..698111b55
--- /dev/null
+++ b/src/buildstream/plugins/elements/import.yaml
@@ -0,0 +1,14 @@
+# The import element simply stages the given sources
+# directly to the root of the sandbox and then collects
+# the output to create an output artifact.
+#
+config:
+
+ # By default we collect everything staged, specify a
+ # directory here to output only a subset of the staged
+ # input sources.
+ source: /
+
+ # Prefix the output with an optional directory, by default
+ # the input is found at the root of the produced artifact.
+ target: /
diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py
new file mode 100644
index 000000000..15ef115d9
--- /dev/null
+++ b/src/buildstream/plugins/elements/junction.py
@@ -0,0 +1,229 @@
+#
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jürg Billeter <juerg.billeter@codethink.co.uk>
+
+"""
+junction - Integrate subprojects
+================================
+This element is a link to another BuildStream project. It allows integration
+of multiple projects into a single pipeline.
+
+Overview
+--------
+
+.. code:: yaml
+
+ kind: junction
+
+ # Specify the BuildStream project source
+ sources:
+ - kind: git
+ url: upstream:projectname.git
+ track: master
+ ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6
+
+ # Specify the junction configuration
+ config:
+
+ # Override project options
+ options:
+ machine_arch: "%{machine_arch}"
+ debug: True
+
+ # Optionally look in a subpath of the source repository for the project
+ path: projects/hello
+
+ # Optionally specify another junction element to serve as a target for
+ # this element. Target should be defined using the syntax
+ # ``{junction-name}:{element-name}``.
+ #
+ # Note that this option cannot be used in conjunction with sources.
+ target: sub-project.bst:sub-sub-project.bst
+
+.. note::
+
+ The configuration option to allow specifying junction targets is available
+ since :ref:`format version 24 <project_format_version>`.
+
+.. note::
+
+ Junction elements may not specify any dependencies as they are simply
+ links to other projects and are not in the dependency graph on their own.
+
+With a junction element in place, local elements can depend on elements in
+the other BuildStream project using the additional ``junction`` attribute in the
+dependency dictionary:
+
+.. code:: yaml
+
+ depends:
+ - junction: toolchain.bst
+ filename: gcc.bst
+ type: build
+
+While junctions are elements, only a limited set of element operations is
+supported. They can be tracked and fetched like other elements.
+However, junction elements do not produce any artifacts, which means that
+they cannot be built or staged. It also means that another element cannot
+depend on a junction element itself.
+
+.. note::
+
+ BuildStream does not implicitly track junction elements. This means
+ that if we were to invoke: `bst build --track-all ELEMENT` on an element
+ which uses a junction element, the ref of the junction element
+ will not automatically be updated if a more recent version exists.
+
+ Therefore, if you require the most up-to-date version of a subproject,
+ you must explicitly track the junction element by invoking:
+ `bst source track JUNCTION_ELEMENT`.
+
+ Furthermore, elements within the subproject are also not tracked by default.
+ For this, we must specify the `--track-cross-junctions` option. This option
+ must be preceeded by `--track ELEMENT` or `--track-all`.
+
+
+Sources
+-------
+``bst show`` does not implicitly fetch junction sources if they haven't been
+cached yet. However, they can be fetched explicitly:
+
+.. code::
+
+ bst source fetch junction.bst
+
+Other commands such as ``bst build`` implicitly fetch junction sources.
+
+Options
+-------
+.. code:: yaml
+
+ options:
+ machine_arch: "%{machine_arch}"
+ debug: True
+
+Junctions can configure options of the linked project. Options are never
+implicitly inherited across junctions, however, variables can be used to
+explicitly assign the same value to a subproject option.
+
+.. _core_junction_nested:
+
+Nested Junctions
+----------------
+Junctions can be nested. That is, subprojects are allowed to have junctions on
+their own. Nested junctions in different subprojects may point to the same
+project, however, in most use cases the same project should be loaded only once.
+BuildStream uses the junction element name as key to determine which junctions
+to merge. It is recommended that the name of a junction is set to the same as
+the name of the linked project.
+
+As the junctions may differ in source version and options, BuildStream cannot
+simply use one junction and ignore the others. Due to this, BuildStream requires
+the user to resolve possibly conflicting nested junctions by creating a junction
+with the same name in the top-level project, which then takes precedence.
+
+Targeting other junctions
+~~~~~~~~~~~~~~~~~~~~~~~~~
+When working with nested junctions, you can also create a junction element that
+targets another junction element in the sub-project. This can be useful if you
+need to ensure that both the top-level project and the sub-project are using
+the same version of the sub-sub-project.
+
+This can be done using the ``target`` configuration option. See below for an
+example:
+
+.. code:: yaml
+
+ kind: junction
+
+ config:
+ target: subproject.bst:subsubproject.bst
+
+In the above example, this junction element would be targeting the junction
+element named ``subsubproject.bst`` in the subproject referred to by
+``subproject.bst``.
+
+Note that when targeting another junction, the names of the junction element
+must not be the same as the name of the target.
+"""
+
+from collections.abc import Mapping
+from buildstream import Element, ElementError
+from buildstream._pipeline import PipelineError
+
+
+# Element implementation for the 'junction' kind.
+class JunctionElement(Element):
+ # pylint: disable=attribute-defined-outside-init
+
+ # Junctions are not allowed any dependencies
+ BST_FORBID_BDEPENDS = True
+ BST_FORBID_RDEPENDS = True
+
+ def configure(self, node):
+ self.path = self.node_get_member(node, str, 'path', default='')
+ self.options = self.node_get_member(node, Mapping, 'options', default={})
+ self.target = self.node_get_member(node, str, 'target', default=None)
+ self.target_element = None
+ self.target_junction = None
+
+ def preflight(self):
+ # "target" cannot be used in conjunction with:
+ # 1. sources
+ # 2. config['options']
+ # 3. config['path']
+ if self.target and any(self.sources()):
+ raise ElementError("junction elements cannot define both 'sources' and 'target' config option")
+ if self.target and any(self.node_items(self.options)):
+ raise ElementError("junction elements cannot define both 'options' and 'target'")
+ if self.target and self.path:
+ raise ElementError("junction elements cannot define both 'path' and 'target'")
+
+ # Validate format of target, if defined
+ if self.target:
+ try:
+ self.target_junction, self.target_element = self.target.split(":")
+ except ValueError:
+ raise ElementError("'target' option must be in format '{junction-name}:{element-name}'")
+
+ # We cannot target a junction that has the same name as us, since that
+ # will cause an infinite recursion while trying to load it.
+ if self.name == self.target_element:
+ raise ElementError("junction elements cannot target an element with the same name")
+
+ def get_unique_key(self):
+ # Junctions do not produce artifacts. get_unique_key() implementation
+ # is still required for `bst source fetch`.
+ return 1
+
+ def configure_sandbox(self, sandbox):
+ raise PipelineError("Cannot build junction elements")
+
+ def stage(self, sandbox):
+ raise PipelineError("Cannot stage junction elements")
+
+ def generate_script(self):
+ raise PipelineError("Cannot build junction elements")
+
+ def assemble(self, sandbox):
+ raise PipelineError("Cannot build junction elements")
+
+
+# Plugin entry point
+def setup():
+ return JunctionElement
diff --git a/src/buildstream/plugins/elements/make.py b/src/buildstream/plugins/elements/make.py
new file mode 100644
index 000000000..67a261100
--- /dev/null
+++ b/src/buildstream/plugins/elements/make.py
@@ -0,0 +1,56 @@
+#
+# Copyright Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Ed Baunton <ebaunton1@bloomberg.net>
+
+"""
+make - Make build element
+=========================
+This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
+using GNU make based build.
+
+.. note::
+
+ The ``make`` element is available since :ref:`format version 9 <project_format_version>`
+
+Here is the default configuration for the ``make`` element in full:
+
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/make.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'make' kind.
+class MakeElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return MakeElement
diff --git a/src/buildstream/plugins/elements/make.yaml b/src/buildstream/plugins/elements/make.yaml
new file mode 100644
index 000000000..83e5c658f
--- /dev/null
+++ b/src/buildstream/plugins/elements/make.yaml
@@ -0,0 +1,42 @@
+# make default configurations
+
+variables:
+ make: make PREFIX="%{prefix}"
+ make-install: make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
+
+ # Set this if the sources cannot handle parallelization.
+ #
+ # notparallel: True
+
+config:
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{make}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{make-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+ MAKEFLAGS: -j%{max-jobs}
+ V: 1
+
+# And dont consider MAKEFLAGS or V as something which may
+# affect build output.
+environment-nocache:
+- MAKEFLAGS
+- V
diff --git a/src/buildstream/plugins/elements/makemaker.py b/src/buildstream/plugins/elements/makemaker.py
new file mode 100644
index 000000000..7da051592
--- /dev/null
+++ b/src/buildstream/plugins/elements/makemaker.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+makemaker - Perl MakeMaker build element
+========================================
+A :mod:`BuildElement <buildstream.buildelement>` implementation for using
+the Perl ExtUtil::MakeMaker build system
+
+The MakeMaker default configuration:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/makemaker.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'makemaker' kind.
+class MakeMakerElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return MakeMakerElement
diff --git a/src/buildstream/plugins/elements/makemaker.yaml b/src/buildstream/plugins/elements/makemaker.yaml
new file mode 100644
index 000000000..c9c4622cb
--- /dev/null
+++ b/src/buildstream/plugins/elements/makemaker.yaml
@@ -0,0 +1,48 @@
+# Default configuration for the Perl ExtUtil::MakeMaker
+# build system
+
+variables:
+
+ # To install perl distributions into the correct location
+ # in our chroot we need to set PREFIX to <destdir>/<prefix>
+ # in the configure-commands.
+ #
+ # The mapping between PREFIX and the final installation
+ # directories is complex and depends upon the configuration
+ # of perl see,
+ # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories
+ # and ExtUtil::MakeMaker's documentation for more details.
+ configure: |
+
+ perl Makefile.PL PREFIX=%{install-root}%{prefix}
+
+ make: make
+ make-install: make install
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands:
+ - |
+ %{configure}
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{make}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{make-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
diff --git a/src/buildstream/plugins/elements/manual.py b/src/buildstream/plugins/elements/manual.py
new file mode 100644
index 000000000..bbda65312
--- /dev/null
+++ b/src/buildstream/plugins/elements/manual.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+manual - Manual build element
+=============================
+The most basic build element does nothing but allows users to
+add custom build commands to the array understood by the :mod:`BuildElement <buildstream.buildelement>`
+
+The empty configuration is as such:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/manual.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'manual' kind.
+class ManualElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return ManualElement
diff --git a/src/buildstream/plugins/elements/manual.yaml b/src/buildstream/plugins/elements/manual.yaml
new file mode 100644
index 000000000..38fe7d163
--- /dev/null
+++ b/src/buildstream/plugins/elements/manual.yaml
@@ -0,0 +1,22 @@
+# Manual build element does not provide any default
+# build commands
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands: []
+
+ # Commands for building the software
+ #
+ build-commands: []
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands: []
+
+ # Commands for stripping installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
diff --git a/src/buildstream/plugins/elements/meson.py b/src/buildstream/plugins/elements/meson.py
new file mode 100644
index 000000000..d80f77977
--- /dev/null
+++ b/src/buildstream/plugins/elements/meson.py
@@ -0,0 +1,71 @@
+# Copyright (C) 2017 Patrick Griffis
+# Copyright (C) 2018 Codethink Ltd.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+meson - Meson build element
+===========================
+This is a :mod:`BuildElement <buildstream.buildelement>` implementation for
+using `Meson <http://mesonbuild.com/>`_ build scripts.
+
+You will often want to pass additional arguments to ``meson``. This should
+be done on a per-element basis by setting the ``meson-local`` variable. Here is
+an example:
+
+.. code:: yaml
+
+ variables:
+ meson-local: |
+ -Dmonkeys=yes
+
+If you want to pass extra options to ``meson`` for every element in your
+project, set the ``meson-global`` variable in your project.conf file. Here is
+an example of that:
+
+.. code:: yaml
+
+ elements:
+ meson:
+ variables:
+ meson-global: |
+ -Dmonkeys=always
+
+Here is the default configuration for the ``meson`` element in full:
+
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/meson.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'meson' kind.
+class MesonElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return MesonElement
diff --git a/src/buildstream/plugins/elements/meson.yaml b/src/buildstream/plugins/elements/meson.yaml
new file mode 100644
index 000000000..2172cb34c
--- /dev/null
+++ b/src/buildstream/plugins/elements/meson.yaml
@@ -0,0 +1,79 @@
+# Meson default configuration
+
+variables:
+
+ build-dir: _builddir
+
+ # Project-wide extra arguments to be passed to `meson`
+ meson-global: ''
+
+ # Element-specific extra arguments to be passed to `meson`.
+ meson-local: ''
+
+ # For backwards compatibility only, do not use.
+ meson-extra: ''
+
+ meson-args: |
+
+ --prefix=%{prefix} \
+ --bindir=%{bindir} \
+ --sbindir=%{sbindir} \
+ --sysconfdir=%{sysconfdir} \
+ --datadir=%{datadir} \
+ --includedir=%{includedir} \
+ --libdir=%{libdir} \
+ --libexecdir=%{libexecdir} \
+ --localstatedir=%{localstatedir} \
+ --sharedstatedir=%{sharedstatedir} \
+ --mandir=%{mandir} \
+ --infodir=%{infodir} %{meson-extra} %{meson-global} %{meson-local}
+
+ meson: meson %{conf-root} %{build-dir} %{meson-args}
+
+ ninja: |
+ ninja -j ${NINJAJOBS} -C %{build-dir}
+
+ ninja-install: |
+ env DESTDIR="%{install-root}" ninja -C %{build-dir} install
+
+ # Set this if the sources cannot handle parallelization.
+ #
+ # notparallel: True
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands:
+ - |
+ %{meson}
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{ninja}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{ninja-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+
+# Use max-jobs CPUs for building
+environment:
+ NINJAJOBS: |
+ %{max-jobs}
+
+# And dont consider NINJAJOBS as something which may
+# affect build output.
+environment-nocache:
+- NINJAJOBS
diff --git a/src/buildstream/plugins/elements/modulebuild.py b/src/buildstream/plugins/elements/modulebuild.py
new file mode 100644
index 000000000..63e3840dc
--- /dev/null
+++ b/src/buildstream/plugins/elements/modulebuild.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+modulebuild - Perl Module::Build build element
+==============================================
+A :mod:`BuildElement <buildstream.buildelement>` implementation for using
+the Perl Module::Build build system
+
+The modulebuild default configuration:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/modulebuild.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'modulebuild' kind.
+class ModuleBuildElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return ModuleBuildElement
diff --git a/src/buildstream/plugins/elements/modulebuild.yaml b/src/buildstream/plugins/elements/modulebuild.yaml
new file mode 100644
index 000000000..18f034bab
--- /dev/null
+++ b/src/buildstream/plugins/elements/modulebuild.yaml
@@ -0,0 +1,48 @@
+# Default configuration for the Perl Module::Build
+# build system.
+
+variables:
+
+ # To install perl distributions into the correct location
+ # in our chroot we need to set PREFIX to <destdir>/<prefix>
+ # in the configure-commands.
+ #
+ # The mapping between PREFIX and the final installation
+ # directories is complex and depends upon the configuration
+ # of perl see,
+ # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories
+ # and ExtUtil::MakeMaker's documentation for more details.
+ configure: |
+
+ perl Build.PL --prefix "%{install-root}%{prefix}"
+
+ perl-build: ./Build
+ perl-install: ./Build install
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands:
+ - |
+ %{configure}
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{perl-build}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{perl-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
diff --git a/src/buildstream/plugins/elements/pip.py b/src/buildstream/plugins/elements/pip.py
new file mode 100644
index 000000000..4a9eefde1
--- /dev/null
+++ b/src/buildstream/plugins/elements/pip.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2017 Mathieu Bridon
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Mathieu Bridon <bochecha@daitauha.fr>
+
+"""
+pip - Pip build element
+=======================
+A :mod:`BuildElement <buildstream.buildelement>` implementation for installing
+Python modules with pip
+
+The pip default configuration:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/pip.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'pip' kind.
+class PipElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return PipElement
diff --git a/src/buildstream/plugins/elements/pip.yaml b/src/buildstream/plugins/elements/pip.yaml
new file mode 100644
index 000000000..294d4ad9a
--- /dev/null
+++ b/src/buildstream/plugins/elements/pip.yaml
@@ -0,0 +1,36 @@
+# Pip default configurations
+
+variables:
+
+ pip: pip
+ pip-flags: |
+ %{pip} install --no-deps --root=%{install-root} --prefix=%{prefix}
+ pip-install-package: |
+ %{pip-flags} %{conf-root}
+ pip-download-dir: |
+ .bst_pip_downloads
+ pip-install-dependencies: |
+ if [ -e %{pip-download-dir} ]; then %{pip-flags} %{pip-download-dir}/*; fi
+
+config:
+
+ configure-commands: []
+ build-commands: []
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{pip-install-package}
+ - |
+ %{pip-install-dependencies}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+ - |
+ %{fix-pyc-timestamps}
diff --git a/src/buildstream/plugins/elements/qmake.py b/src/buildstream/plugins/elements/qmake.py
new file mode 100644
index 000000000..56a0e641e
--- /dev/null
+++ b/src/buildstream/plugins/elements/qmake.py
@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+qmake - QMake build element
+===========================
+A :mod:`BuildElement <buildstream.buildelement>` implementation for using
+the qmake build system
+
+The qmake default configuration:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/qmake.yaml
+ :language: yaml
+
+See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
+details on common configuration options for build elements.
+"""
+
+from buildstream import BuildElement, SandboxFlags
+
+
+# Element implementation for the 'qmake' kind.
+class QMakeElement(BuildElement):
+ # Supports virtual directories (required for remote execution)
+ BST_VIRTUAL_DIRECTORY = True
+
+ # Enable command batching across prepare() and assemble()
+ def configure_sandbox(self, sandbox):
+ super().configure_sandbox(sandbox)
+ self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY,
+ collect=self.get_variable('install-root'))
+
+
+# Plugin entry point
+def setup():
+ return QMakeElement
diff --git a/src/buildstream/plugins/elements/qmake.yaml b/src/buildstream/plugins/elements/qmake.yaml
new file mode 100644
index 000000000..4ac31932e
--- /dev/null
+++ b/src/buildstream/plugins/elements/qmake.yaml
@@ -0,0 +1,50 @@
+# QMake default configuration
+
+variables:
+
+ qmake: qmake -makefile %{conf-root}
+ make: make
+ make-install: make -j1 INSTALL_ROOT="%{install-root}" install
+
+ # Set this if the sources cannot handle parallelization.
+ #
+ # notparallel: True
+
+config:
+
+ # Commands for configuring the software
+ #
+ configure-commands:
+ - |
+ %{qmake}
+
+ # Commands for building the software
+ #
+ build-commands:
+ - |
+ %{make}
+
+ # Commands for installing the software into a
+ # destination folder
+ #
+ install-commands:
+ - |
+ %{make-install}
+
+ # Commands for stripping debugging information out of
+ # installed binaries
+ #
+ strip-commands:
+ - |
+ %{strip-binaries}
+
+# Use max-jobs CPUs for building and enable verbosity
+environment:
+ MAKEFLAGS: -j%{max-jobs}
+ V: 1
+
+# And dont consider MAKEFLAGS or V as something which may
+# affect build output.
+environment-nocache:
+- MAKEFLAGS
+- V
diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py
new file mode 100644
index 000000000..0d194dcc1
--- /dev/null
+++ b/src/buildstream/plugins/elements/script.py
@@ -0,0 +1,69 @@
+#
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+# Jonathan Maw <jonathan.maw@codethink.co.uk>
+
+"""
+script - Run scripts to create output
+=====================================
+This element allows one to run some commands to mutate the
+input and create some output.
+
+.. note::
+
+ Script elements may only specify build dependencies. See
+ :ref:`the format documentation <format_dependencies>` for more
+ detail on specifying dependencies.
+
+The default configuration and possible options are as such:
+ .. literalinclude:: ../../../src/buildstream/plugins/elements/script.yaml
+ :language: yaml
+"""
+
+import buildstream
+
+
+# Element implementation for the 'script' kind.
+class ScriptElement(buildstream.ScriptElement):
+ # pylint: disable=attribute-defined-outside-init
+
+ # This plugin has been modified to avoid the use of Sandbox.get_directory
+ BST_VIRTUAL_DIRECTORY = True
+
+ def configure(self, node):
+ for n in self.node_get_member(node, list, 'layout', []):
+ dst = self.node_subst_member(n, 'destination')
+ elm = self.node_subst_member(n, 'element', None)
+ self.layout_add(elm, dst)
+
+ self.node_validate(node, [
+ 'commands', 'root-read-only', 'layout'
+ ])
+
+ cmds = self.node_subst_list(node, "commands")
+ self.add_commands("commands", cmds)
+
+ self.set_work_dir()
+ self.set_install_root()
+ self.set_root_read_only(self.node_get_member(node, bool,
+ 'root-read-only', False))
+
+
+# Plugin entry point
+def setup():
+ return ScriptElement
diff --git a/src/buildstream/plugins/elements/script.yaml b/src/buildstream/plugins/elements/script.yaml
new file mode 100644
index 000000000..b388378da
--- /dev/null
+++ b/src/buildstream/plugins/elements/script.yaml
@@ -0,0 +1,25 @@
+# Common script element variables
+variables:
+ # Defines the directory commands will be run from.
+ cwd: /
+
+# Script element configuration
+config:
+
+ # Defines whether to run the sandbox with '/' read-only.
+ # It is recommended to set root as read-only wherever possible.
+ root-read-only: False
+
+ # Defines where to stage elements which are direct or indirect dependencies.
+ # By default, all direct dependencies are staged to '/'.
+ # This is also commonly used to take one element as an environment
+ # containing the tools used to operate on the other element.
+ # layout:
+ # - element: foo-tools.bst
+ # destination: /
+ # - element: foo-system.bst
+ # destination: %{build-root}
+
+ # List of commands to run in the sandbox.
+ commands: []
+
diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py
new file mode 100644
index 000000000..97517ca48
--- /dev/null
+++ b/src/buildstream/plugins/elements/stack.py
@@ -0,0 +1,66 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+stack - Symbolic Element for dependency grouping
+================================================
+Stack elements are simply a symbolic element used for representing
+a logical group of elements.
+"""
+
+from buildstream import Element
+
+
+# Element implementation for the 'stack' kind.
+class StackElement(Element):
+
+ # This plugin has been modified to avoid the use of Sandbox.get_directory
+ BST_VIRTUAL_DIRECTORY = True
+
+ def configure(self, node):
+ pass
+
+ def preflight(self):
+ pass
+
+ def get_unique_key(self):
+ # We do not add anything to the build, only our dependencies
+ # do, so our unique key is just a constant.
+ return 1
+
+ def configure_sandbox(self, sandbox):
+ pass
+
+ def stage(self, sandbox):
+ pass
+
+ def assemble(self, sandbox):
+
+ # Just create a dummy empty artifact, its existence is a statement
+ # that all this stack's dependencies are built.
+ vrootdir = sandbox.get_virtual_directory()
+ vrootdir.descend('output', create=True)
+
+ # And we're done
+ return '/output'
+
+
+# Plugin entry point
+def setup():
+ return StackElement
diff --git a/src/buildstream/plugins/sources/__init__.py b/src/buildstream/plugins/sources/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/buildstream/plugins/sources/__init__.py
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
new file mode 100644
index 000000000..b9b15e268
--- /dev/null
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -0,0 +1,250 @@
+"""A base abstract class for source implementations which download a file"""
+
+import os
+import urllib.request
+import urllib.error
+import contextlib
+import shutil
+import netrc
+
+from buildstream import Source, SourceError, Consistency
+from buildstream import utils
+
+
+class _NetrcFTPOpener(urllib.request.FTPHandler):
+
+ def __init__(self, netrc_config):
+ self.netrc = netrc_config
+
+ def _split(self, netloc):
+ userpass, hostport = urllib.parse.splituser(netloc)
+ host, port = urllib.parse.splitport(hostport)
+ if userpass:
+ user, passwd = urllib.parse.splitpasswd(userpass)
+ else:
+ user = None
+ passwd = None
+ return host, port, user, passwd
+
+ def _unsplit(self, host, port, user, passwd):
+ if port:
+ host = '{}:{}'.format(host, port)
+ if user:
+ if passwd:
+ user = '{}:{}'.format(user, passwd)
+ host = '{}@{}'.format(user, host)
+
+ return host
+
+ def ftp_open(self, req):
+ host, port, user, passwd = self._split(req.host)
+
+ if user is None and self.netrc:
+ entry = self.netrc.authenticators(host)
+ if entry:
+ user, _, passwd = entry
+
+ req.host = self._unsplit(host, port, user, passwd)
+
+ return super().ftp_open(req)
+
+
+class _NetrcPasswordManager:
+
+ def __init__(self, netrc_config):
+ self.netrc = netrc_config
+
+ def add_password(self, realm, uri, user, passwd):
+ pass
+
+ def find_user_password(self, realm, authuri):
+ if not self.netrc:
+ return None, None
+ parts = urllib.parse.urlsplit(authuri)
+ entry = self.netrc.authenticators(parts.hostname)
+ if not entry:
+ return None, None
+ else:
+ login, _, password = entry
+ return login, password
+
+
+class DownloadableFileSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+
+ COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+
+ __urlopener = None
+
+ def configure(self, node):
+ self.original_url = self.node_get_member(node, str, 'url')
+ self.ref = self.node_get_member(node, str, 'ref', None)
+ self.url = self.translate_url(self.original_url)
+ self._warn_deprecated_etag(node)
+
+ def preflight(self):
+ return
+
+ def get_unique_key(self):
+ return [self.original_url, self.ref]
+
+ def get_consistency(self):
+ if self.ref is None:
+ return Consistency.INCONSISTENT
+
+ if os.path.isfile(self._get_mirror_file()):
+ return Consistency.CACHED
+
+ else:
+ return Consistency.RESOLVED
+
+ def load_ref(self, node):
+ self.ref = self.node_get_member(node, str, 'ref', None)
+ self._warn_deprecated_etag(node)
+
+ def get_ref(self):
+ return self.ref
+
+ def set_ref(self, ref, node):
+ node['ref'] = self.ref = ref
+
+ def track(self):
+ # there is no 'track' field in the source to determine what/whether
+ # or not to update refs, because tracking a ref is always a conscious
+ # decision by the user.
+ with self.timed_activity("Tracking {}".format(self.url),
+ silent_nested=True):
+ new_ref = self._ensure_mirror()
+
+ if self.ref and self.ref != new_ref:
+ detail = "When tracking, new ref differs from current ref:\n" \
+ + " Tracked URL: {}\n".format(self.url) \
+ + " Current ref: {}\n".format(self.ref) \
+ + " New ref: {}\n".format(new_ref)
+ self.warn("Potential man-in-the-middle attack!", detail=detail)
+
+ return new_ref
+
+ def fetch(self):
+
+ # Just a defensive check, it is impossible for the
+ # file to be already cached because Source.fetch() will
+ # not be called if the source is already Consistency.CACHED.
+ #
+ if os.path.isfile(self._get_mirror_file()):
+ return # pragma: nocover
+
+ # Download the file, raise hell if the sha256sums don't match,
+ # and mirror the file otherwise.
+ with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
+ sha256 = self._ensure_mirror()
+ if sha256 != self.ref:
+ raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
+ .format(self.url, sha256, self.ref))
+
+ def _warn_deprecated_etag(self, node):
+ etag = self.node_get_member(node, str, 'etag', None)
+ if etag:
+ provenance = self.node_provenance(node, member_name='etag')
+ self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
+
+ def _get_etag(self, ref):
+ etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
+ if os.path.exists(etagfilename):
+ with open(etagfilename, 'r') as etagfile:
+ return etagfile.read()
+
+ return None
+
+ def _store_etag(self, ref, etag):
+ etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
+ with utils.save_file_atomic(etagfilename) as etagfile:
+ etagfile.write(etag)
+
+ def _ensure_mirror(self):
+ # Downloads from the url and caches it according to its sha256sum.
+ try:
+ with self.tempdir() as td:
+ default_name = os.path.basename(self.url)
+ request = urllib.request.Request(self.url)
+ request.add_header('Accept', '*/*')
+
+ # We do not use etag in case what we have in cache is
+ # not matching ref in order to be able to recover from
+ # corrupted download.
+ if self.ref:
+ etag = self._get_etag(self.ref)
+
+ # Do not re-download the file if the ETag matches.
+ if etag and self.get_consistency() == Consistency.CACHED:
+ request.add_header('If-None-Match', etag)
+
+ opener = self.__get_urlopener()
+ with contextlib.closing(opener.open(request)) as response:
+ info = response.info()
+
+ etag = info['ETag'] if 'ETag' in info else None
+
+ filename = info.get_filename(default_name)
+ filename = os.path.basename(filename)
+ local_file = os.path.join(td, filename)
+ with open(local_file, 'wb') as dest:
+ shutil.copyfileobj(response, dest)
+
+ # Make sure url-specific mirror dir exists.
+ if not os.path.isdir(self._get_mirror_dir()):
+ os.makedirs(self._get_mirror_dir())
+
+ # Store by sha256sum
+ sha256 = utils.sha256sum(local_file)
+ # Even if the file already exists, move the new file over.
+ # In case the old file was corrupted somehow.
+ os.rename(local_file, self._get_mirror_file(sha256))
+
+ if etag:
+ self._store_etag(sha256, etag)
+ return sha256
+
+ except urllib.error.HTTPError as e:
+ if e.code == 304:
+ # 304 Not Modified.
+ # Because we use etag only for matching ref, currently specified ref is what
+ # we would have downloaded.
+ return self.ref
+ raise SourceError("{}: Error mirroring {}: {}"
+ .format(self, self.url, e), temporary=True) from e
+
+ except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError, ValueError) as e:
+ # Note that urllib.request.Request in the try block may throw a
+ # ValueError for unknown url types, so we handle it here.
+ raise SourceError("{}: Error mirroring {}: {}"
+ .format(self, self.url, e), temporary=True) from e
+
+ def _get_mirror_dir(self):
+ return os.path.join(self.get_mirror_directory(),
+ utils.url_directory_name(self.original_url))
+
+ def _get_mirror_file(self, sha=None):
+ return os.path.join(self._get_mirror_dir(), sha or self.ref)
+
+ def __get_urlopener(self):
+ if not DownloadableFileSource.__urlopener:
+ try:
+ netrc_config = netrc.netrc()
+ except OSError:
+ # If the .netrc file was not found, FileNotFoundError will be
+ # raised, but OSError will be raised directly by the netrc package
+ # in the case that $HOME is not set.
+ #
+ # This will catch both cases.
+ #
+ DownloadableFileSource.__urlopener = urllib.request.build_opener()
+ except netrc.NetrcParseError as e:
+ self.warn('{}: While reading .netrc: {}'.format(self, e))
+ return urllib.request.build_opener()
+ else:
+ netrc_pw_mgr = _NetrcPasswordManager(netrc_config)
+ http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr)
+ ftp_handler = _NetrcFTPOpener(netrc_config)
+ DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler)
+ return DownloadableFileSource.__urlopener
diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py
new file mode 100644
index 000000000..e59986da6
--- /dev/null
+++ b/src/buildstream/plugins/sources/bzr.py
@@ -0,0 +1,210 @@
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jonathan Maw <jonathan.maw@codethink.co.uk>
+
+"""
+bzr - stage files from a bazaar repository
+==========================================
+
+**Host dependencies:**
+
+ * bzr
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the bzr source kind
+ kind: bzr
+
+ # Specify the bzr url. Bazaar URLs come in many forms, see
+ # `bzr help urlspec` for more information. Using an alias defined
+ # in your project configuration is encouraged.
+ url: https://launchpad.net/bzr
+
+ # Specify the tracking branch. This is mandatory, as bzr cannot identify
+ # an individual revision outside its branch. bzr URLs that omit the branch
+ # name implicitly specify the trunk branch, but bst requires this to be
+ # explicit.
+ track: trunk
+
+ # Specify the ref. This is a revision number. This is usually a decimal,
+ # but revisions on a branch are of the form
+ # <revision-branched-from>.<branch-number>.<revision-since-branching>
+ # e.g. 6622.1.6.
+ # The ref must be specified to build, and 'bst source track' will update the
+ # revision number to the one on the tip of the branch specified in 'track'.
+ ref: 6622
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+"""
+
+import os
+import shutil
+import fcntl
+from contextlib import contextmanager
+
+from buildstream import Source, SourceError, Consistency
+from buildstream import utils
+
+
+class BzrSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ self.node_validate(node, ['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS])
+
+ self.original_url = self.node_get_member(node, str, 'url')
+ self.tracking = self.node_get_member(node, str, 'track')
+ self.ref = self.node_get_member(node, str, 'ref', None)
+ self.url = self.translate_url(self.original_url)
+
+ def preflight(self):
+ # Check if bzr is installed, get the binary at the same time.
+ self.host_bzr = utils.get_host_tool('bzr')
+
+ def get_unique_key(self):
+ return [self.original_url, self.tracking, self.ref]
+
+ def get_consistency(self):
+ if self.ref is None or self.tracking is None:
+ return Consistency.INCONSISTENT
+
+ # Lock for the _check_ref()
+ with self._locked():
+ if self._check_ref():
+ return Consistency.CACHED
+ else:
+ return Consistency.RESOLVED
+
+ def load_ref(self, node):
+ self.ref = self.node_get_member(node, str, 'ref', None)
+
+ def get_ref(self):
+ return self.ref
+
+ def set_ref(self, ref, node):
+ node['ref'] = self.ref = ref
+
+ def track(self):
+ with self.timed_activity("Tracking {}".format(self.url),
+ silent_nested=True), self._locked():
+ self._ensure_mirror(skip_ref_check=True)
+ ret, out = self.check_output([self.host_bzr, "version-info",
+ "--custom", "--template={revno}",
+ self._get_branch_dir()],
+ fail="Failed to read the revision number at '{}'"
+ .format(self._get_branch_dir()))
+ if ret != 0:
+ raise SourceError("{}: Failed to get ref for tracking {}".format(self, self.tracking))
+
+ return out
+
+ def fetch(self):
+ with self.timed_activity("Fetching {}".format(self.url),
+ silent_nested=True), self._locked():
+ self._ensure_mirror()
+
+ def stage(self, directory):
+ self.call([self.host_bzr, "checkout", "--lightweight",
+ "--revision=revno:{}".format(self.ref),
+ self._get_branch_dir(), directory],
+ fail="Failed to checkout revision {} from branch {} to {}"
+ .format(self.ref, self._get_branch_dir(), directory))
+ # Remove .bzr dir
+ shutil.rmtree(os.path.join(directory, ".bzr"))
+
+ def init_workspace(self, directory):
+ url = os.path.join(self.url, self.tracking)
+ with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
+ # Checkout from the cache
+ self.call([self.host_bzr, "branch",
+ "--use-existing-dir",
+ "--revision=revno:{}".format(self.ref),
+ self._get_branch_dir(), directory],
+ fail="Failed to branch revision {} from branch {} to {}"
+ .format(self.ref, self._get_branch_dir(), directory))
+ # Switch the parent branch to the source's origin
+ self.call([self.host_bzr, "switch",
+ "--directory={}".format(directory), url],
+ fail="Failed to switch workspace's parent branch to {}".format(url))
+
+ # _locked()
+ #
+ # This context manager ensures exclusive access to the
+ # bzr repository.
+ #
+ @contextmanager
+ def _locked(self):
+ lockdir = os.path.join(self.get_mirror_directory(), 'locks')
+ lockfile = os.path.join(
+ lockdir,
+ utils.url_directory_name(self.original_url) + '.lock'
+ )
+ os.makedirs(lockdir, exist_ok=True)
+ with open(lockfile, 'w') as lock:
+ fcntl.flock(lock, fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(lock, fcntl.LOCK_UN)
+
+ def _check_ref(self):
+ # If the mirror doesnt exist yet, then we dont have the ref
+ if not os.path.exists(self._get_branch_dir()):
+ return False
+
+ return self.call([self.host_bzr, "revno",
+ "--revision=revno:{}".format(self.ref),
+ self._get_branch_dir()]) == 0
+
+ def _get_branch_dir(self):
+ return os.path.join(self._get_mirror_dir(), self.tracking)
+
+ def _get_mirror_dir(self):
+ return os.path.join(self.get_mirror_directory(),
+ utils.url_directory_name(self.original_url))
+
+ def _ensure_mirror(self, skip_ref_check=False):
+ mirror_dir = self._get_mirror_dir()
+ bzr_metadata_dir = os.path.join(mirror_dir, ".bzr")
+ if not os.path.exists(bzr_metadata_dir):
+ self.call([self.host_bzr, "init-repo", "--no-trees", mirror_dir],
+ fail="Failed to initialize bzr repository")
+
+ branch_dir = os.path.join(mirror_dir, self.tracking)
+ branch_url = self.url + "/" + self.tracking
+ if not os.path.exists(branch_dir):
+ # `bzr branch` the branch if it doesn't exist
+ # to get the upstream code
+ self.call([self.host_bzr, "branch", branch_url, branch_dir],
+ fail="Failed to branch from {} to {}".format(branch_url, branch_dir))
+
+ else:
+ # `bzr pull` the branch if it does exist
+ # to get any changes to the upstream code
+ self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url],
+ fail="Failed to pull new changes for {}".format(branch_dir))
+
+ if not skip_ref_check and not self._check_ref():
+ raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref),
+ reason="ref-not-mirrored")
+
+
+def setup():
+ return BzrSource
diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py
new file mode 100644
index 000000000..e45994951
--- /dev/null
+++ b/src/buildstream/plugins/sources/deb.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Phillip Smyth <phillip.smyth@codethink.co.uk>
+# Jonathan Maw <jonathan.maw@codethink.co.uk>
+# Richard Maw <richard.maw@codethink.co.uk>
+
+"""
+deb - stage files from .deb packages
+====================================
+
+**Host dependencies:**
+
+ * arpy (python package)
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the deb source kind
+ kind: deb
+
+ # Specify the deb url. Using an alias defined in your project
+ # configuration is encouraged. 'bst source track' will update the
+ # sha256sum in 'ref' to the downloaded file's sha256sum.
+ url: upstream:foo.deb
+
+ # Specify the ref. It's a sha256sum of the file you download.
+ ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
+
+ # Specify the basedir to return only the specified dir and its children
+ base-dir: ''
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+"""
+
+import tarfile
+from contextlib import contextmanager
+import arpy # pylint: disable=import-error
+
+from .tar import TarSource
+
+
+class DebSource(TarSource):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ super().configure(node)
+
+ self.base_dir = self.node_get_member(node, str, 'base-dir', None)
+
+ def preflight(self):
+ return
+
+ @contextmanager
+ def _get_tar(self):
+ with open(self._get_mirror_file(), 'rb') as deb_file:
+ arpy_archive = arpy.Archive(fileobj=deb_file)
+ arpy_archive.read_all_headers()
+ data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0]
+ # ArchiveFileData is not enough like a file object for tarfile to use.
+ # Monkey-patching a seekable method makes it close enough for TarFile to open.
+ data_tar_arpy.seekable = lambda *args: True
+ tar = tarfile.open(fileobj=data_tar_arpy, mode="r:*")
+ yield tar
+
+
+def setup():
+ return DebSource
diff --git a/src/buildstream/plugins/sources/git.py b/src/buildstream/plugins/sources/git.py
new file mode 100644
index 000000000..5e6834979
--- /dev/null
+++ b/src/buildstream/plugins/sources/git.py
@@ -0,0 +1,168 @@
+#
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""
+git - stage files from a git repository
+=======================================
+
+**Host dependencies:**
+
+ * git
+
+.. attention::
+
+ Note that this plugin **will checkout git submodules by default**; even if
+ they are not specified in the `.bst` file.
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the git source kind
+ kind: git
+
+ # Specify the repository url, using an alias defined
+ # in your project configuration is recommended.
+ url: upstream:foo.git
+
+ # Optionally specify a symbolic tracking branch or tag, this
+ # will be used to update the 'ref' when refreshing the pipeline.
+ track: master
+
+ # Optionally specify the ref format used for tracking.
+ # The default is 'sha1' for the raw commit hash.
+ # If you specify 'git-describe', the commit hash will be prefixed
+ # with the closest tag.
+ ref-format: sha1
+
+ # Specify the commit ref, this must be specified in order to
+ # checkout sources and build, but can be automatically updated
+ # if the 'track' attribute was specified.
+ ref: d63cbb6fdc0bbdadc4a1b92284826a6d63a7ebcd
+
+ # Optionally specify whether submodules should be checked-out.
+ # If not set, this will default to 'True'
+ checkout-submodules: True
+
+ # If your repository has submodules, explicitly specifying the
+ # url from which they are to be fetched allows you to easily
+ # rebuild the same sources from a different location. This is
+ # especially handy when used with project defined aliases which
+ # can be redefined at a later time.
+ # You may also explicitly specify whether to check out this
+ # submodule. If 'checkout' is set, it will override
+ # 'checkout-submodules' with the value set below.
+ submodules:
+ plugins/bar:
+ url: upstream:bar.git
+ checkout: True
+ plugins/baz:
+ url: upstream:baz.git
+ checkout: False
+
+ # Enable tag tracking.
+ #
+ # This causes the `tags` metadata to be populated automatically
+ # as a result of tracking the git source.
+ #
+ # By default this is 'False'.
+ #
+ track-tags: True
+
+ # If the list of tags below is set, then a lightweight dummy
+ # git repository will be staged along with the content at
+ # build time.
+ #
+ # This is useful for a growing number of modules which use
+ # `git describe` at build time in order to determine the version
+ # which will be encoded into the built software.
+ #
+ # The 'tags' below is considered as a part of the git source
+ # reference and will be stored in the 'project.refs' file if
+ # that has been selected as your project's ref-storage.
+ #
+ # Migration notes:
+ #
+ # If you are upgrading from BuildStream 1.2, which used to
+ # stage the entire repository by default, you will notice that
+ # some modules which use `git describe` are broken, and will
+ # need to enable this feature in order to fix them.
+ #
+ # If you need to enable this feature without changing the
+ # the specific commit that you are building, then we recommend
+ # the following migration steps for any git sources where
+ # `git describe` is required:
+ #
+ # o Enable `track-tags` feature
+ # o Set the `track` parameter to the desired commit sha which
+ # the current `ref` points to
+ # o Run `bst source track` for these elements, this will result in
+ # populating the `tags` portion of the refs without changing
+ # the refs
+ # o Restore the `track` parameter to the branches which you have
+ # previously been tracking afterwards.
+ #
+ tags:
+ - tag: lightweight-example
+ commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78
+ annotated: false
+ - tag: annotated-example
+ commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a
+ annotated: true
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+
+**Configurable Warnings:**
+
+This plugin provides the following :ref:`configurable warnings <configurable_warnings>`:
+
+- ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never
+ added with `git submodule add`.
+
+- ``git:unlisted-submodule`` - A submodule is present in the git repository but was not specified in
+ the source configuration and was not disabled for checkout.
+
+ .. note::
+
+ The ``git:unlisted-submodule`` warning is available since :ref:`format version 20 <project_format_version>`
+
+- ``git:invalid-submodule`` - A submodule is specified in the source configuration but does not exist
+ in the repository.
+
+ .. note::
+
+ The ``git:invalid-submodule`` warning is available since :ref:`format version 20 <project_format_version>`
+
+This plugin also utilises the following configurable :class:`core warnings <buildstream.types.CoreWarnings>`:
+
+- :attr:`ref-not-in-track <buildstream.types.CoreWarnings.REF_NOT_IN_TRACK>` - The provided ref was not
+ found in the provided track in the element's git repository.
+"""
+
+from buildstream import _GitSourceBase
+
+
+class GitSource(_GitSourceBase):
+ pass
+
+
+# Plugin entry point
+def setup():
+ return GitSource
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
new file mode 100644
index 000000000..50df85427
--- /dev/null
+++ b/src/buildstream/plugins/sources/local.py
@@ -0,0 +1,147 @@
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+# Tiago Gomes <tiago.gomes@codethink.co.uk>
+
+"""
+local - stage local files and directories
+=========================================
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the local source kind
+ kind: local
+
+ # Specify the project relative path to a file or directory
+ path: files/somefile.txt
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+"""
+
+import os
+import stat
+from buildstream import Source, Consistency
+from buildstream import utils
+
+
+class LocalSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+
+ def __init__(self, context, project, meta):
+ super().__init__(context, project, meta)
+
+ # Cached unique key to avoid multiple file system traversal if the unique key is requested multiple times.
+ self.__unique_key = None
+
+ def configure(self, node):
+ self.node_validate(node, ['path', *Source.COMMON_CONFIG_KEYS])
+ self.path = self.node_get_project_path(node, 'path')
+ self.fullpath = os.path.join(self.get_project_directory(), self.path)
+
+ def preflight(self):
+ pass
+
+ def get_unique_key(self):
+ if self.__unique_key is None:
+ # Get a list of tuples of the the project relative paths and fullpaths
+ if os.path.isdir(self.fullpath):
+ filelist = utils.list_relative_paths(self.fullpath)
+ filelist = [(relpath, os.path.join(self.fullpath, relpath)) for relpath in filelist]
+ else:
+ filelist = [(self.path, self.fullpath)]
+
+ # Return a list of (relative filename, sha256 digest) tuples, a sorted list
+ # has already been returned by list_relative_paths()
+ self.__unique_key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
+ return self.__unique_key
+
+ def get_consistency(self):
+ return Consistency.CACHED
+
+ # We dont have a ref, we're a local file...
+ def load_ref(self, node):
+ pass
+
+ def get_ref(self):
+ return None # pragma: nocover
+
+ def set_ref(self, ref, node):
+ pass # pragma: nocover
+
+ def fetch(self):
+ # Nothing to do here for a local source
+ pass # pragma: nocover
+
+ def stage(self, directory):
+
+ # Dont use hardlinks to stage sources, they are not write protected
+ # in the sandbox.
+ with self.timed_activity("Staging local files at {}".format(self.path)):
+
+ if os.path.isdir(self.fullpath):
+ files = list(utils.list_relative_paths(self.fullpath))
+ utils.copy_files(self.fullpath, directory)
+ else:
+ destfile = os.path.join(directory, os.path.basename(self.path))
+ files = [os.path.basename(self.path)]
+ utils.safe_copy(self.fullpath, destfile)
+
+ for f in files:
+ # Non empty directories are not listed by list_relative_paths
+ dirs = f.split(os.sep)
+ for i in range(1, len(dirs)):
+ d = os.path.join(directory, *(dirs[:i]))
+ assert os.path.isdir(d) and not os.path.islink(d)
+ os.chmod(d, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+
+ path = os.path.join(directory, f)
+ if os.path.islink(path):
+ pass
+ elif os.path.isdir(path):
+ os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+ else:
+ st = os.stat(path)
+ if st.st_mode & stat.S_IXUSR:
+ os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
+ else:
+ os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+
+ def _get_local_path(self):
+ return self.fullpath
+
+
+# Create a unique key for a file
+def unique_key(filename):
+
+ # Return some hard coded things for files which
+ # have no content to calculate a key for
+ if os.path.islink(filename):
+ # For a symbolic link, use the link target as its unique identifier
+ return os.readlink(filename)
+ elif os.path.isdir(filename):
+ return "0"
+
+ return utils.sha256sum(filename)
+
+
+# Plugin entry point
+def setup():
+ return LocalSource
diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py
new file mode 100644
index 000000000..e42868264
--- /dev/null
+++ b/src/buildstream/plugins/sources/patch.py
@@ -0,0 +1,101 @@
+#
+# Copyright Bloomberg Finance LP
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Chandan Singh <csingh43@bloomberg.net>
+# Tiago Gomes <tiago.gomes@codethink.co.uk>
+
+"""
+patch - apply locally stored patches
+====================================
+
+**Host dependencies:**
+
+ * patch
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the local source kind
+ kind: patch
+
+ # Specify the project relative path to a patch file
+ path: files/somefile.diff
+
+ # Optionally specify the strip level, defaults to 1
+ strip-level: 1
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+"""
+
+import os
+from buildstream import Source, SourceError, Consistency
+from buildstream import utils
+
+
+class PatchSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+
+ BST_REQUIRES_PREVIOUS_SOURCES_STAGE = True
+
+ def configure(self, node):
+ self.path = self.node_get_project_path(node, 'path',
+ check_is_file=True)
+ self.strip_level = self.node_get_member(node, int, "strip-level", 1)
+ self.fullpath = os.path.join(self.get_project_directory(), self.path)
+
+ def preflight(self):
+ # Check if patch is installed, get the binary at the same time
+ self.host_patch = utils.get_host_tool("patch")
+
+ def get_unique_key(self):
+ return [self.path, utils.sha256sum(self.fullpath), self.strip_level]
+
+ def get_consistency(self):
+ return Consistency.CACHED
+
+ def load_ref(self, node):
+ pass
+
+ def get_ref(self):
+ return None # pragma: nocover
+
+ def set_ref(self, ref, node):
+ pass # pragma: nocover
+
+ def fetch(self):
+ # Nothing to do here for a local source
+ pass # pragma: nocover
+
+ def stage(self, directory):
+ with self.timed_activity("Applying local patch: {}".format(self.path)):
+
+ # Bail out with a comprehensive message if the target directory is empty
+ if not os.listdir(directory):
+ raise SourceError("Nothing to patch in directory '{}'".format(directory),
+ reason="patch-no-files")
+
+ strip_level_option = "-p{}".format(self.strip_level)
+ self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory],
+ fail="Failed to apply patch {}".format(self.path))
+
+
+# Plugin entry point
+def setup():
+ return PatchSource
diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py
new file mode 100644
index 000000000..9d6c40d74
--- /dev/null
+++ b/src/buildstream/plugins/sources/pip.py
@@ -0,0 +1,254 @@
+#
+# Copyright 2018 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Chandan Singh <csingh43@bloomberg.net>
+
+"""
+pip - stage python packages using pip
+=====================================
+
+**Host depndencies:**
+
+ * ``pip`` python module
+
+This plugin will download source distributions for specified packages using
+``pip`` but will not install them. It is expected that the elements using this
+source will install the downloaded packages.
+
+Downloaded tarballs will be stored in a directory called ".bst_pip_downloads".
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the pip source kind
+ kind: pip
+
+ # Optionally specify index url, defaults to PyPi
+ # This url is used to discover new versions of packages and download them
+ # Projects intending to mirror their sources to a permanent location should
+ # use an aliased url, and declare the alias in the project configuration
+ url: https://mypypi.example.com/simple
+
+ # Optionally specify the path to requirements files
+ # Note that either 'requirements-files' or 'packages' must be defined
+ requirements-files:
+ - requirements.txt
+
+ # Optionally specify a list of additional packages
+ # Note that either 'requirements-files' or 'packages' must be defined
+ packages:
+ - flake8
+
+ # Specify the ref. It is a list of strings of format
+ # "<package-name>==<version>", separated by "\\n".
+ # Usually this will be contents of a requirements.txt file where all
+ # package versions have been frozen.
+ ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+
+.. note::
+
+ The ``pip`` plugin is available since :ref:`format version 16 <project_format_version>`
+"""
+
+import hashlib
+import os
+import re
+
+from buildstream import Consistency, Source, SourceError, utils
+
+_OUTPUT_DIRNAME = '.bst_pip_downloads'
+_PYPI_INDEX_URL = 'https://pypi.org/simple/'
+
+# Used only for finding pip command
+_PYTHON_VERSIONS = [
+ 'python', # when running in a venv, we might not have the exact version
+ 'python2.7',
+ 'python3.0',
+ 'python3.1',
+ 'python3.2',
+ 'python3.3',
+ 'python3.4',
+ 'python3.5',
+ 'python3.6',
+ 'python3.7',
+]
+
+# List of allowed extensions taken from
+# https://docs.python.org/3/distutils/sourcedist.html.
+# Names of source distribution archives must be of the form
+# '%{package-name}-%{version}.%{extension}'.
+_SDIST_RE = re.compile(
+ r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
+ re.IGNORECASE)
+
+
+class PipSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+
+ # We need access to previous sources at track time to use requirements.txt
+ # but not at fetch time as self.ref should contain sufficient information
+ # for this plugin
+ BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
+
+ def configure(self, node):
+ self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] +
+ Source.COMMON_CONFIG_KEYS)
+ self.ref = self.node_get_member(node, str, 'ref', None)
+ self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL)
+ self.index_url = self.translate_url(self.original_url)
+ self.packages = self.node_get_member(node, list, 'packages', [])
+ self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])
+
+ if not (self.packages or self.requirements_files):
+ raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))
+
+ def preflight(self):
+ # Try to find a pip version that supports download command
+ self.host_pip = None
+ for python in reversed(_PYTHON_VERSIONS):
+ try:
+ host_python = utils.get_host_tool(python)
+ rc = self.call([host_python, '-m', 'pip', 'download', '--help'])
+ if rc == 0:
+ self.host_pip = [host_python, '-m', 'pip']
+ break
+ except utils.ProgramNotFoundError:
+ pass
+
+ if self.host_pip is None:
+ raise SourceError("{}: Unable to find a suitable pip command".format(self))
+
+ def get_unique_key(self):
+ return [self.original_url, self.ref]
+
+ def get_consistency(self):
+ if not self.ref:
+ return Consistency.INCONSISTENT
+ if os.path.exists(self._mirror) and os.listdir(self._mirror):
+ return Consistency.CACHED
+ return Consistency.RESOLVED
+
+ def get_ref(self):
+ return self.ref
+
+ def load_ref(self, node):
+ self.ref = self.node_get_member(node, str, 'ref', None)
+
+ def set_ref(self, ref, node):
+ node['ref'] = self.ref = ref
+
+ def track(self, previous_sources_dir):
+ # XXX pip does not offer any public API other than the CLI tool so it
+ # is not feasible to correctly parse the requirements file or to check
+ # which package versions pip is going to install.
+ # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
+ # for details.
+ # As a result, we have to wastefully install the packages during track.
+ with self.tempdir() as tmpdir:
+ install_args = self.host_pip + ['download',
+ '--no-binary', ':all:',
+ '--index-url', self.index_url,
+ '--dest', tmpdir]
+ for requirement_file in self.requirements_files:
+ fpath = os.path.join(previous_sources_dir, requirement_file)
+ install_args += ['-r', fpath]
+ install_args += self.packages
+
+ self.call(install_args, fail="Failed to install python packages")
+ reqs = self._parse_sdist_names(tmpdir)
+
+ return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
+
+ def fetch(self):
+ with self.tempdir() as tmpdir:
+ packages = self.ref.strip().split('\n')
+ package_dir = os.path.join(tmpdir, 'packages')
+ os.makedirs(package_dir)
+ self.call([*self.host_pip,
+ 'download',
+ '--no-binary', ':all:',
+ '--index-url', self.index_url,
+ '--dest', package_dir,
+ *packages],
+ fail="Failed to install python packages: {}".format(packages))
+
+ # If the mirror directory already exists, assume that some other
+ # process has fetched the sources before us and ensure that we do
+ # not raise an error in that case.
+ try:
+ utils.move_atomic(package_dir, self._mirror)
+ except utils.DirectoryExistsError:
+ # Another process has beaten us and has fetched the sources
+ # before us.
+ pass
+ except OSError as e:
+ raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}"
+ .format(self, package_dir, self._mirror, e)) from e
+
+ def stage(self, directory):
+ with self.timed_activity("Staging Python packages", silent_nested=True):
+ utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME))
+
+ # Directory where this source should stage its files
+ #
+ @property
+ def _mirror(self):
+ if not self.ref:
+ return None
+ return os.path.join(self.get_mirror_directory(),
+ utils.url_directory_name(self.original_url),
+ hashlib.sha256(self.ref.encode()).hexdigest())
+
+ # Parse names of downloaded source distributions
+ #
+ # Args:
+ # basedir (str): Directory containing source distribution archives
+ #
+ # Returns:
+ # (list): List of (package_name, version) tuples in sorted order
+ #
+ def _parse_sdist_names(self, basedir):
+ reqs = []
+ for f in os.listdir(basedir):
+ pkg = _match_package_name(f)
+ if pkg is not None:
+ reqs.append(pkg)
+
+ return sorted(reqs)
+
+
+# Extract the package name and version of a source distribution
+#
+# Args:
+# filename (str): Filename of the source distribution
+#
+# Returns:
+# (tuple): A tuple of (package_name, version)
+#
+def _match_package_name(filename):
+ pkg_match = _SDIST_RE.match(filename)
+ if pkg_match is None:
+ return None
+ return pkg_match.groups()
+
+
+def setup():
+ return PipSource
diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py
new file mode 100644
index 000000000..562a8f226
--- /dev/null
+++ b/src/buildstream/plugins/sources/remote.py
@@ -0,0 +1,93 @@
+#
+# Copyright Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Ed Baunton <ebaunton1@bloomberg.net>
+
+"""
+remote - stage files from remote urls
+=====================================
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the remote source kind
+ kind: remote
+
+ # Optionally specify a relative staging filename.
+ # If not specified, the basename of the url will be used.
+ # filename: customfilename
+
+ # Optionally specify whether the downloaded file should be
+ # marked executable.
+ # executable: true
+
+ # Specify the url. Using an alias defined in your project
+ # configuration is encouraged. 'bst source track' will update the
+ # sha256sum in 'ref' to the downloaded file's sha256sum.
+ url: upstream:foo
+
+ # Specify the ref. It's a sha256sum of the file you download.
+ ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+
+.. note::
+
+ The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>`
+"""
+import os
+from buildstream import SourceError, utils
+from ._downloadablefilesource import DownloadableFileSource
+
+
+class RemoteSource(DownloadableFileSource):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ super().configure(node)
+
+ self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
+ self.executable = self.node_get_member(node, bool, 'executable', False)
+
+ if os.sep in self.filename:
+ raise SourceError('{}: filename parameter cannot contain directories'.format(self),
+ reason="filename-contains-directory")
+ self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable'])
+
+ def get_unique_key(self):
+ return super().get_unique_key() + [self.filename, self.executable]
+
+ def stage(self, directory):
+ # Same as in local plugin, don't use hardlinks to stage sources, they
+ # are not write protected in the sandbox.
+ dest = os.path.join(directory, self.filename)
+ with self.timed_activity("Staging remote file to {}".format(dest)):
+
+ utils.safe_copy(self._get_mirror_file(), dest)
+
+ # To prevent user's umask introducing variability here, explicitly set
+ # file modes.
+ if self.executable:
+ os.chmod(dest, 0o755)
+ else:
+ os.chmod(dest, 0o644)
+
+
+def setup():
+ return RemoteSource
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
new file mode 100644
index 000000000..31dc17497
--- /dev/null
+++ b/src/buildstream/plugins/sources/tar.py
@@ -0,0 +1,202 @@
+#
+# Copyright (C) 2017 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Jonathan Maw <jonathan.maw@codethink.co.uk>
+
+"""
+tar - stage files from tar archives
+===================================
+
+**Host dependencies:**
+
+ * lzip (for .tar.lz files)
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the tar source kind
+ kind: tar
+
+ # Specify the tar url. Using an alias defined in your project
+ # configuration is encouraged. 'bst source track' will update the
+ # sha256sum in 'ref' to the downloaded file's sha256sum.
+ url: upstream:foo.tar
+
+ # Specify the ref. It's a sha256sum of the file you download.
+ ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
+
+ # Specify a glob pattern to indicate the base directory to extract
+ # from the tarball. The first matching directory will be used.
+ #
+ # Note that this is '*' by default since most standard release
+ # tarballs contain a self named subdirectory at the root which
+ # contains the files one normally wants to extract to build.
+ #
+ # To extract the root of the tarball directly, this can be set
+ # to an empty string.
+ base-dir: '*'
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+"""
+
+import os
+import tarfile
+from contextlib import contextmanager
+from tempfile import TemporaryFile
+
+from buildstream import SourceError
+from buildstream import utils
+
+from ._downloadablefilesource import DownloadableFileSource
+
+
+class TarSource(DownloadableFileSource):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ super().configure(node)
+
+ self.base_dir = self.node_get_member(node, str, 'base-dir', '*') or None
+
+ self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
+
+ def preflight(self):
+ self.host_lzip = None
+ if self.url.endswith('.lz'):
+ self.host_lzip = utils.get_host_tool('lzip')
+
+ def get_unique_key(self):
+ return super().get_unique_key() + [self.base_dir]
+
+ @contextmanager
+ def _run_lzip(self):
+ assert self.host_lzip
+ with TemporaryFile() as lzip_stdout:
+ with open(self._get_mirror_file(), 'r') as lzip_file:
+ self.call([self.host_lzip, '-d'],
+ stdin=lzip_file,
+ stdout=lzip_stdout)
+
+ lzip_stdout.seek(0, 0)
+ yield lzip_stdout
+
+ @contextmanager
+ def _get_tar(self):
+ if self.url.endswith('.lz'):
+ with self._run_lzip() as lzip_dec:
+ with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
+ yield tar
+ else:
+ with tarfile.open(self._get_mirror_file()) as tar:
+ yield tar
+
+ def stage(self, directory):
+ try:
+ with self._get_tar() as tar:
+ base_dir = None
+ if self.base_dir:
+ base_dir = self._find_base_dir(tar, self.base_dir)
+
+ if base_dir:
+ tar.extractall(path=directory, members=self._extract_members(tar, base_dir))
+ else:
+ tar.extractall(path=directory)
+
+ except (tarfile.TarError, OSError) as e:
+ raise SourceError("{}: Error staging source: {}".format(self, e)) from e
+
+ # Override and translate which filenames to extract
+ def _extract_members(self, tar, base_dir):
+ if not base_dir.endswith(os.sep):
+ base_dir = base_dir + os.sep
+
+ L = len(base_dir)
+ for member in tar.getmembers():
+
+ # First, ensure that a member never starts with `./`
+ if member.path.startswith('./'):
+ member.path = member.path[2:]
+
+ # Now extract only the paths which match the normalized path
+ if member.path.startswith(base_dir):
+
+ # If it's got a link name, give it the same treatment, we
+ # need the link targets to match up with what we are staging
+ #
+ # NOTE: Its possible this is not perfect, we may need to
+ # consider links which point outside of the chosen
+ # base directory.
+ #
+ if member.type == tarfile.LNKTYPE:
+ member.linkname = member.linkname[L:]
+
+ member.path = member.path[L:]
+ yield member
+
+ # We want to iterate over all paths of a tarball, but getmembers()
+ # is not enough because some tarballs simply do not contain the leading
+ # directory paths for the archived files.
+ def _list_tar_paths(self, tar):
+
+ visited = set()
+ for member in tar.getmembers():
+
+ # Remove any possible leading './', offer more consistent behavior
+ # across tarballs encoded with or without a leading '.'
+ member_name = member.name.lstrip('./')
+
+ if not member.isdir():
+
+ # Loop over the components of a path, for a path of a/b/c/d
+ # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding
+ # the final component
+ components = member_name.split('/')
+ for i in range(len(components) - 1):
+ dir_component = '/'.join([components[j] for j in range(i + 1)])
+ if dir_component not in visited:
+ visited.add(dir_component)
+ try:
+ # Dont yield directory members which actually do
+ # exist in the archive
+ _ = tar.getmember(dir_component)
+ except KeyError:
+ if dir_component != '.':
+ yield dir_component
+
+ continue
+
+ # Avoid considering the '.' directory, if any is included in the archive
+ # this is to avoid the default 'base-dir: *' value behaving differently
+ # depending on whether the tarball was encoded with a leading '.' or not
+ elif member_name == '.':
+ continue
+
+ yield member_name
+
+ def _find_base_dir(self, tar, pattern):
+ paths = self._list_tar_paths(tar)
+ matches = sorted(list(utils.glob(paths, pattern)))
+ if not matches:
+ raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern))
+
+ return matches[0]
+
+
+def setup():
+ return TarSource
diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py
new file mode 100644
index 000000000..03efcef79
--- /dev/null
+++ b/src/buildstream/plugins/sources/zip.py
@@ -0,0 +1,181 @@
+#
+# Copyright (C) 2017 Mathieu Bridon
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Mathieu Bridon <bochecha@daitauha.fr>
+
+"""
+zip - stage files from zip archives
+===================================
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the zip source kind
+ kind: zip
+
+ # Specify the zip url. Using an alias defined in your project
+ # configuration is encouraged. 'bst source track' will update the
+ # sha256sum in 'ref' to the downloaded file's sha256sum.
+ url: upstream:foo.zip
+
+ # Specify the ref. It's a sha256sum of the file you download.
+ ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
+
+ # Specify a glob pattern to indicate the base directory to extract
+ # from the archive. The first matching directory will be used.
+ #
+ # Note that this is '*' by default since most standard release
+ # archives contain a self named subdirectory at the root which
+ # contains the files one normally wants to extract to build.
+ #
+ # To extract the root of the archive directly, this can be set
+ # to an empty string.
+ base-dir: '*'
+
+See :ref:`built-in functionality doumentation <core_source_builtins>` for
+details on common configuration options for sources.
+
+.. attention::
+
+ File permissions are not preserved. All extracted directories have
+ permissions 0755 and all extracted files have permissions 0644.
+"""
+
+import os
+import zipfile
+import stat
+
+from buildstream import SourceError
+from buildstream import utils
+
+from ._downloadablefilesource import DownloadableFileSource
+
+
+class ZipSource(DownloadableFileSource):
+ # pylint: disable=attribute-defined-outside-init
+
+ def configure(self, node):
+ super().configure(node)
+
+ self.base_dir = self.node_get_member(node, str, 'base-dir', '*') or None
+
+ self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
+
+ def get_unique_key(self):
+ return super().get_unique_key() + [self.base_dir]
+
+ def stage(self, directory):
+ exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH)
+ noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+ try:
+ with zipfile.ZipFile(self._get_mirror_file()) as archive:
+ base_dir = None
+ if self.base_dir:
+ base_dir = self._find_base_dir(archive, self.base_dir)
+
+ if base_dir:
+ members = self._extract_members(archive, base_dir)
+ else:
+ members = archive.namelist()
+
+ for member in members:
+ written = archive.extract(member, path=directory)
+
+ # zipfile.extract might create missing directories
+ rel = os.path.relpath(written, start=directory)
+ assert not os.path.isabs(rel)
+ rel = os.path.dirname(rel)
+ while rel:
+ os.chmod(os.path.join(directory, rel), exec_rights)
+ rel = os.path.dirname(rel)
+
+ if os.path.islink(written):
+ pass
+ elif os.path.isdir(written):
+ os.chmod(written, exec_rights)
+ else:
+ os.chmod(written, noexec_rights)
+
+ except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e:
+ raise SourceError("{}: Error staging source: {}".format(self, e)) from e
+
+ # Override and translate which filenames to extract
+ def _extract_members(self, archive, base_dir):
+ if not base_dir.endswith(os.sep):
+ base_dir = base_dir + os.sep
+
+ L = len(base_dir)
+ for member in archive.infolist():
+ if member.filename == base_dir:
+ continue
+
+ if member.filename.startswith(base_dir):
+ member.filename = member.filename[L:]
+ yield member
+
+ # We want to iterate over all paths of an archive, but namelist()
+ # is not enough because some archives simply do not contain the leading
+ # directory paths for the archived files.
+ def _list_archive_paths(self, archive):
+
+ visited = {}
+ for member in archive.infolist():
+
+ # ZipInfo.is_dir() is only available in python >= 3.6, but all
+ # it does is check for a trailing '/' in the name
+ #
+ if not member.filename.endswith('/'):
+
+ # Loop over the components of a path, for a path of a/b/c/d
+ # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding
+ # the final component
+ components = member.filename.split('/')
+ for i in range(len(components) - 1):
+ dir_component = '/'.join([components[j] for j in range(i + 1)])
+ if dir_component not in visited:
+ visited[dir_component] = True
+ try:
+ # Dont yield directory members which actually do
+ # exist in the archive
+ _ = archive.getinfo(dir_component)
+ except KeyError:
+ if dir_component != '.':
+ yield dir_component
+
+ continue
+
+ # Avoid considering the '.' directory, if any is included in the archive
+ # this is to avoid the default 'base-dir: *' value behaving differently
+ # depending on whether the archive was encoded with a leading '.' or not
+ elif member.filename == '.' or member.filename == './':
+ continue
+
+ yield member.filename
+
+ def _find_base_dir(self, archive, pattern):
+ paths = self._list_archive_paths(archive)
+ matches = sorted(list(utils.glob(paths, pattern)))
+ if not matches:
+ raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern))
+
+ return matches[0]
+
+
+def setup():
+ return ZipSource