Handle removed files from integration in compose plugin

Fixes issue #147

2 files changed, 58 insertions, 20 deletions

diff --git a/buildstream/element.py b/buildstream/element.py
index 3f48cc84a..c4ebbea8d 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -338,6 +338,34 @@ class Element(Plugin):
         value = self.node_get_list_element(node, str, member_name, indices)
         return self.__variables.subst(value)
 
+    def compute_manifest(self, *, include=None, exclude=None, orphans=True):
+        """Compute and return this element's manifest
+
+        The manifest consists on the list of file paths in the
+        artifact.  The files in the manifest are selected according to
+        `include`, `exclude` and `orphans` parameters. If `include` is
+        not specified then all files spoken for by any domain are
+        included unless explicitly excluded with an `exclude` domain.
+
+        The main use is to stage all files of the artifact, then
+        execute integration commands, and finally keep only files from
+        the manifest. This is useful if integration commands are
+        expected to run on all files. In that case, call to
+        :meth:`stage_artifact` should not have any `include`,
+        `exclude` and `orphans` parameters. Those should be passed to
+        `compute_manifest` instead.
+
+        Args:
+           include (list): An optional list of domains to include files from
+           exclude (list): An optional list of domains to exclude files from
+           orphans (bool): Whether to include files not spoken for by split domains
+
+        Yields:
+           (str): The paths of the files in manifest
+        """
+        self._assert_cached()
+        return self.__compute_splits(include, exclude, orphans)
+
     def stage_artifact(self, sandbox, *, path=None, include=None, exclude=None, orphans=True):
         """Stage this element's output artifact in the sandbox
 
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 00a77722b..2d6a94d47 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -91,18 +91,29 @@ class ComposeElement(Element):
 
     def assemble(self, sandbox):
 
+        require_split = self.include or self.exclude or not self.include_orphans
+
         # Stage deps in the sandbox root
         with self.timed_activity("Staging dependencies", silent_nested=True):
             self.stage_dependency_artifacts(sandbox, Scope.BUILD)
 
+        split = set()
+        if require_split:
+            with self.timed_activity("Computing split", silent_nested=True):
+                for dep in self.dependencies(Scope.BUILD):
+                    files = dep.compute_manifest(self.include,
+                                                 self.exclude,
+                                                 self.include_orphans)
+                    split.update(files)
+
         # Make a snapshot of all the files.
         basedir = sandbox.get_directory()
         snapshot = {
             f: getmtime(os.path.join(basedir, f))
             for f in utils.list_relative_paths(basedir)
         }
-        manifest = []
         integration_files = []
+        removed_files = []
 
         # Run any integration commands provided by the dependencies
         # once they are all staged and ready
@@ -111,26 +122,32 @@ class ComposeElement(Element):
                 for dep in self.dependencies(Scope.BUILD):
                     dep.integrate(sandbox)
 
-                integration_files = [
-                    path for path in utils.list_relative_paths(basedir)
-                    if (snapshot.get(path) is None or
-                        snapshot[path] != getmtime(os.path.join(basedir, path)))
-                ]
-                self.info("Integration effected {} files".format(len(integration_files)))
-
-        manifest += integration_files
+                if require_split:
+                    integration_files = [
+                        path for path in utils.list_relative_paths(basedir)
+                        if (snapshot.get(path) is None or
+                            snapshot[path] != getmtime(os.path.join(basedir, path)))
+                    ]
+                    removed_files = [
+                        path for path in split
+                        if not os.path.lexists(os.path.join(basedir, path))
+                    ]
+                    self.info("Integration effected {} files and removed {} files"
+                              .format(len(integration_files), len(removed_files)))
 
         # The remainder of this is expensive, make an early exit if
         # we're not being selective about what is to be included.
-        if not (self.include or self.exclude) and self.include_orphans:
+        if not require_split:
             return '/'
 
+        split.update(integration_files)
+        split.difference_update(removed_files)
+
         # XXX We should be moving things outside of the build sandbox
         # instead of into a subdir. The element assemble() method should
         # support this in some way.
         #
         installdir = os.path.join(basedir, 'buildstream', 'install')
-        stagedir = os.path.join(os.sep, 'buildstream', 'install')
         os.makedirs(installdir, exist_ok=True)
 
         # We already saved the manifest for created files in the integration phase,
@@ -154,15 +171,8 @@ class ComposeElement(Element):
         detail = "\n".join(lines)
 
         with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
-            self.stage_dependency_artifacts(sandbox, Scope.BUILD,
-                                            path=stagedir,
-                                            include=self.include,
-                                            exclude=self.exclude,
-                                            orphans=self.include_orphans)
-
-            if self.integration:
-                self.status("Moving {} integration files".format(len(integration_files)))
-                utils.move_files(basedir, installdir, files=integration_files)
+            self.info("Composing {} files".format(len(split)))
+            utils.link_files(basedir, installdir, split)
 
         # And we're done
         return os.path.join(os.sep, 'buildstream', 'install')