element.py: Added assemble() abstract method and some counterparts

  o Added the assemble() method which plugins implement to assemble
    an output artifact.

  o Added stage() method to stage the element's artifact into a sandbox

  o Added stage_sources() method to stage the element's sources in a sandbox

  o Added internal _assemble() method to act as controller for the plugin
    assemble() method, this takes care of instantiating the sandbox and
    collecting the output.

1 files changed, 159 insertions, 13 deletions

diff --git a/buildstream/element.py b/buildstream/element.py
index 159a5b5ea..c0d92e8ae 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -23,10 +23,13 @@ import copy
 import inspect
 from contextlib import contextmanager
 from enum import Enum
+import tempfile
+import shutil
 
 from . import _yaml
 from ._variables import Variables
-from . import LoadError, LoadErrorReason
+from . import LoadError, LoadErrorReason, ElementError
+from . import Sandbox
 from . import Plugin
 from . import utils
 
@@ -206,6 +209,86 @@ class Element(Plugin):
         value = self.node_get_list_element(node, str, member_name, indices)
         return self.__variables.subst(value)
 
+    def stage(self, sandbox, path=None):
+        """Stage this element's output in the sandbox
+
+        Args:
+           sandbox (:class:`.Sandbox`): The build sandbox
+           path (str): An optional sandbox relative path
+
+        Raises:
+           (:class:`.ElementError`): If the element output does not exist
+
+        **Example:**
+
+        .. code:: python
+
+          # Stage the dependencies for a build of 'self'
+          for dep in self.dependencies(Scope.BUILD):
+              dep.stage(sandbox)
+        """
+        project = self.get_project()
+        key = self._get_cache_key()
+
+        # Time to use the artifact, check once more that it's there
+        self._cached(recalculate=True, assert_cached=True)
+
+        with self.timed_activity("Staging {}/{}/{}".format(project.name, self.name, key)):
+            # Get the extracted artifact
+            artifact = self.__artifacts.extract(project.name, self.name, key)
+
+            # Hard link it into the staging area
+            #
+            # XXX For now assuming that it's a read-only location
+            basedir = sandbox.executor.fs_root
+            stagedir = basedir \
+                if path is None \
+                else os.path.join(basedir, path.lstrip(os.sep))
+            utils.link_files(artifact, stagedir)
+
+    def stage_sources(self, sandbox, path=None):
+        """Stage this element's source input
+
+        Args:
+           sandbox (:class:`.Sandbox`): The build sandbox
+           path (str): An optional sandbox relative path
+
+        **Example:**
+
+        .. code:: python
+
+          # Stage the sources of 'self' to the /build directory
+          # in the sandbox
+          self.stage_sources(sandbox, '/build')
+        """
+        basedir = sandbox.executor.fs_root
+        stagedir = basedir \
+            if path is None \
+            else os.path.join(basedir, path.lstrip(os.sep))
+        for source in self.__sources:
+            source._stage(stagedir)
+
+    #############################################################
+    #                  Abstract Element Methods                 #
+    #############################################################
+    def assemble(self, sandbox):
+        """Assemble the output artifact
+
+        Args:
+           sandbox (:class:`.Sandbox`): The build sandbox
+
+        Returns:
+           (str): A sandbox relative path to collect
+
+        Raises:
+           (:class:`.ElementError`): When the element raises an error
+
+        Elements must implement this method to create an output
+        artifact from it's sources and dependencies.
+        """
+        raise ImplError("element plugin '{kind}' does not implement assemble()".format(
+            kind=self.get_kind()))
+
     #############################################################
     #            Private Methods used in BuildStream            #
     #############################################################
@@ -267,23 +350,30 @@ class Element(Plugin):
     #
     # Args:
     #    recalculate (bool): Whether to recalculate the cached state again
+    #    assert_cached (bool): Whether to raise an exception if the artifact is missing
     #
     # Returns:
     #    (bool): Whether this element is already present in
     #            the artifact cache
     #
-    def _cached(self, recalculate=False):
+    def _cached(self, recalculate=False, assert_cached=False):
 
-        if self._inconsistent():
-            return False
+        if not self._inconsistent():
+            project = self.get_project()
+            key = self._get_cache_key()
 
-        project = self.get_project()
-        key = self._get_cache_key()
+            if self.__cached is None or recalculate:
+                self.__cached = self.__artifacts.contains(project.name, self.name, key)
 
-        if self.__cached is None or recalculate:
-            self.__cached = self.__artifacts.contains(project.name, self.name, key)
+        if assert_cached and not self.__cached:
+            raise ElementError("{element}: Missing artifact {project}/{name}/{key}"
+                               .format(element=self,
+                                       project=project.name,
+                                       name=self.name,
+                                       key=key))
 
-        return self.__cached
+        # Return False for inconsistent state and retain the None value
+        return False if self._inconsistent() else self.__cached
 
     # _buildable():
     #
@@ -354,6 +444,62 @@ class Element(Plugin):
 
         return changed
 
+    # _assemble():
+    #
+    # Internal method for calling public abstract assemble() method.
+    #
+    # Returns:
+    #    (bool): True if something was assembled, False if the item was cached
+    def _assemble(self):
+
+        # No need to assemble
+        if self._cached(self):
+            return False
+
+        context = self.get_context()
+        with self._output_file() as output_file:
+
+            # Explicitly clean it up, keep the build dir around if exceptions are raised
+            os.makedirs(context.builddir, exist_ok=True)
+            rootdir = tempfile.mkdtemp(prefix="{}-".format(self.name), dir=context.builddir)
+
+            environment = utils._node_sanitize(self.__environment)
+            sandbox = Sandbox(fs_root=rootdir,
+                              env=environment,
+                              stdout=output_file,
+                              stderr=output_file)
+
+            # sandbox.executor.debug = True
+
+            # root is temp directory
+            sandbox.executor.root_ro = False
+            os.mkdir(os.path.join(rootdir, 'tmp'))
+
+            mounts = []
+            mounts.append({'dest': '/dev', 'type': 'host-dev'})
+            mounts.append({'dest': '/proc', 'type': 'proc'})
+            sandbox.set_mounts(mounts)
+
+            # Call the abstract plugin method
+            collect = self.assemble(sandbox)
+
+            # Note important use of lstrip() here
+            collectdir = os.path.join(rootdir, collect.lstrip(os.sep))
+
+            # At this point, we expect an exception was raised leading to
+            # an error message, or we have good output to collect.
+            project = self.get_project()
+            key = self._get_cache_key()
+            self.__artifacts.commit(project.name,
+                                    self.name,
+                                    key,
+                                    collectdir)
+
+            # Finally cleanup the build dir
+            shutil.rmtree(rootdir)
+
+        return True
+
     # _logfile()
     #
     # Compose the log file for this action & pid.
@@ -408,10 +554,6 @@ class Element(Plugin):
             yield fullpath
             self._set_log_handle(None)
 
-    #############################################################
-    #                   Private Local Methods                   #
-    #############################################################
-
     # Override plugin _set_log_handle(), set it for our sources too
     #
     def _set_log_handle(self, logfile):
@@ -419,6 +561,10 @@ class Element(Plugin):
         for source in self._sources():
             source._set_log_handle(logfile)
 
+    #############################################################
+    #                   Private Local Methods                   #
+    #############################################################
+
     def __init_defaults(self):
 
         # Defaults are loaded once per class and then reused