Added new script element.

This allows quite manual transformations of input, allowing
one to select a base for the shell and tooling to use and
stage the input separately in order to create some output.

3 files changed, 184 insertions, 0 deletions

diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py
new file mode 100644
index 000000000..d0bdba2d4
--- /dev/null
+++ b/buildstream/plugins/elements/script.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2017 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+"""Script element
+
+This element allows one to run some commands to mutate the
+input and create some output.
+
+As with build elements, the output created by a script element is
+collected from the ``%{install-root}`` directory.
+
+The default configuration and possible options are as such:
+  .. literalinclude:: ../../../buildstream/plugins/elements/script.yaml
+     :language: yaml
+"""
+
+import os
+from buildstream import utils
+from buildstream import Element, ElementError, Scope
+from buildstream import SandboxFlags
+
+
+# Element implementation for the 'script' kind.
+class ScriptElement(Element):
+
+    def configure(self, node):
+        self.base_dep = self.node_get_member(node, str, 'base')
+        self.input_dep = self.node_get_member(node, str, 'input')
+        self.stage_mode = self.node_get_member(node, str, 'stage-mode')
+
+        # Assert stage mode is valid
+        if self.stage_mode not in ['build', 'install']:
+            p = self.node_provenance(node, 'stage-mode')
+            raise ElementError("{}: Stage mode must be either 'build' or 'install'"
+                               .format(p))
+
+        # Collect variable substituted commands
+        self.commands = []
+        command_list = self.node_get_member(node, list, 'commands', default_value=[])
+        for i in range(len(command_list)):
+            self.commands.append(
+                self.node_subst_list_element(node, 'commands', [i])
+            )
+
+        # To be resolved in preflight when the pipeline is built
+        self.base_elt = None
+        self.input_elt = None
+
+    def preflight(self):
+
+        # Assert that the user did not list any runtime dependencies
+        runtime_deps = list(self.dependencies(Scope.RUN, recurse=False))
+        if runtime_deps:
+            raise ElementError("{}: Only build type dependencies supported by script elements"
+                               .format(self))
+
+        # Assert that the user did not specify any sources, as they will
+        # be ignored by this element type anyway
+        sources = list(self.sources())
+        if sources:
+            raise ElementError("Script elements may not have sources")
+
+        # Assert that a base and an input were specified
+        if not self.base_dep:
+            raise ElementError("{}: No base dependencies were specified".format(self))
+        if not self.input_dep:
+            raise ElementError("{}: No input dependencies were specified".format(self))
+
+        # Now resolve the base and input elements
+        self.base_elt = self.search(Scope.BUILD, self.base_dep)
+        self.input_elt = self.search(Scope.BUILD, self.input_dep)
+
+        if self.base_elt is None:
+            raise ElementError("{}: Could not find base dependency {}".format(self, self.base_dep))
+        if self.input_elt is None:
+            raise ElementError("{}: Could not find input dependency {}".format(self, self.input_dep))
+
+    def get_unique_key(self):
+        return {
+            'commands': self.commands,
+            'base': self.base_dep,
+            'input': self.input_dep,
+            'stage-mode': self.stage_mode
+        }
+
+    def assemble(self, sandbox):
+
+        directory = sandbox.get_directory()
+        environment = self.get_environment()
+
+        # Stage the base in the sandbox root
+        with self.timed_activity("Staging {} as base".format(self.base_dep), silent_nested=True):
+            self.base_elt.stage_dependencies(sandbox, Scope.BUILD)
+
+        # Run any integration commands on the base
+        with self.timed_activity("Integrating sandbox", silent_nested=True):
+            for dep in self.base_elt.dependencies(Scope.BUILD):
+                dep.integrate(sandbox)
+
+        # Ensure some directories we'll need
+        os.makedirs(os.path.join(directory,
+                                 'buildstream',
+                                 'build'), exist_ok=True)
+        os.makedirs(os.path.join(directory,
+                                 'buildstream',
+                                 'install'), exist_ok=True)
+
+        # Stage the input
+        input_dir = os.path.join(os.sep, 'buildstream', self.stage_mode)
+        with self.timed_activity("Staging {} as input at {}"
+                                 .format(self.input_dep, input_dir), silent_nested=True):
+            self.input_elt.stage_dependencies(sandbox, Scope.BUILD, path=input_dir)
+
+        # Run the scripts
+        with self.timed_activity("Running script commands"):
+            for cmd in self.commands:
+                self.status("Running command", detail=cmd)
+
+                # Note the -e switch to 'sh' means to exit with an error
+                # if any untested command fails.
+                exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
+                                       SandboxFlags.ROOT_READ_ONLY,
+                                       cwd=input_dir,
+                                       env=environment)
+                if exitcode != 0:
+                    raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))
+
+        # Return the install dir
+        return os.path.join(os.sep, 'buildstream', 'install')
+
+
+# Plugin entry point
+def setup():
+    return ScriptElement
diff --git a/buildstream/plugins/elements/script.yaml b/buildstream/plugins/elements/script.yaml
new file mode 100644
index 000000000..c21e115fa
--- /dev/null
+++ b/buildstream/plugins/elements/script.yaml
@@ -0,0 +1,32 @@
+
+# Script element configuration
+config:
+
+  # A dependency of this element to use as the root filesystem
+  # for scripting, expressed as a project relative element bst
+  # filename.
+  #
+  # This will define what tools are available to use in
+  # the commands. At minimum, a shell should exist in
+  # the base dependencies in order to run any commands.
+  #
+  #   base: foo.bst
+
+  # A dependency of this element to manipulate as input,
+  # expressed as a project relative element bst filename
+  #
+  #   input: bar.bst
+
+  # Defines where to stage the 'input' dependencies, the
+  # working directory for running commands will also be
+  # set to this directory.
+  #
+  # build:   Stage the input in %{build-root}
+  #
+  # install: Stage the input directly at %{install-root}
+  #
+  stage-mode: build
+
+  # List of commands to run in the sandbox
+  #
+  commands: []
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 1be66fb58..58af4903a 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -40,6 +40,7 @@ General Elements
 * :mod:`stack` - Symbolic Element for dependency grouping
 * :mod:`import` - Import sources directly
 * :mod:`compose` - Compose the output of multiple elements
+* :mod:`script` - Run scripts to create output
 
 Build Elements
 ~~~~~~~~~~~~~~