WIP: add manifest.yaml to artifact metadatasam/manifests

7 files changed, 39 insertions, 20 deletions

diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py
index 824d21fd0..799c2a347 100644
--- a/buildstream/buildelement.py
+++ b/buildstream/buildelement.py
@@ -211,7 +211,7 @@ class BuildElement(Element):
 
         # Return the payload, this is configurable but is generally
         # always the /buildstream/install directory
-        return self.get_variable('install-root')
+        return self.get_variable('install-root'), {}
 
     def _get_commands(self, node, name):
         list_node = self.node_get_member(node, list, name, default_value=[])
diff --git a/buildstream/element.py b/buildstream/element.py
index d2976cfe2..e140163f9 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -382,8 +382,9 @@ class Element(Plugin):
                 if path is None \
                 else os.path.join(basedir, path.lstrip(os.sep))
 
-            files = self.__compute_splits(include, exclude, orphans)
-            result = utils.link_files(artifact, stagedir, files=files)
+            manifest = self.__compute_splits(include, exclude, orphans)
+            result = utils.link_files(artifact, stagedir, files=manifest.keys())
+            result.manifest = manifest
 
         return result
 
@@ -410,6 +411,7 @@ class Element(Plugin):
         """
         overwrites = {}
         ignored = {}
+        manifest = {}
         for dep in self.dependencies(scope):
             result = dep.stage_artifact(sandbox,
                                         path=path,
@@ -421,6 +423,10 @@ class Element(Plugin):
             if result.ignored:
                 ignored[dep.name] = result.ignored
 
+            manifest.update({
+                f:info for f, info in result.manifest.items()
+                if f not in result.ignored})
+
         if overwrites:
             detail = "Staged files overwrite existing files in staging area:\n"
             for key, value in overwrites.items():
@@ -435,6 +441,8 @@ class Element(Plugin):
                 detail += "  " + "  ".join(["/" + f + "\n" for f in value])
             self.warn("Ignored files", detail=detail)
 
+        return manifest
+
     def integrate(self, sandbox):
         """Integrate currently staged filesystem against this artifact.
 
@@ -1083,7 +1091,7 @@ class Element(Plugin):
                     # Step 2 - Stage
                     self.stage(sandbox)
                     # Step 3 - Assemble
-                    collect = self.assemble(sandbox)
+                    collect, manifest = self.assemble(sandbox)
                 except BstError as e:
                     # If an error occurred assembling an element in a sandbox,
                     # then tack on the sandbox directory to the error
@@ -1138,6 +1146,8 @@ class Element(Plugin):
                 }
                 _yaml.dump(_yaml.node_sanitize(meta), os.path.join(metadir, 'artifact.yaml'))
 
+                _yaml.dump(manifest, os.path.join(metadir, 'manifest.yaml'))
+
                 with self.timed_activity("Caching Artifact"):
                     self.__artifacts.commit(self, assembledir)
 
@@ -1609,9 +1619,8 @@ class Element(Plugin):
 
         # No splitting requested, just report complete artifact
         if orphans and not (include or exclude):
-            for filename in utils.list_relative_paths(basedir):
-                yield filename
-            return
+            return {filename: (self.name,)
+                    for filename in utils.list_relative_paths(basedir)}
 
         if not self.__splits:
             self.__init_splits()
@@ -1636,6 +1645,7 @@ class Element(Plugin):
             for filename in utils.list_relative_paths(basedir)
         ]
 
+        manifest = {}
         for filename in element_files:
             include_file = False
             exclude_file = False
@@ -1653,7 +1663,10 @@ class Element(Plugin):
                 include_file = True
 
             if include_file and not exclude_file:
-                yield filename.lstrip(os.sep)
+                claimed_domain = domain if claimed_file else None
+                manifest[filename.lstrip(os.sep)] = (self.name, claimed_domain)
+
+        return manifest
 
     def _load_public_data(self):
         self._assert_cached(recalculate=False)
diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py
index 9d701c72f..224306130 100644
--- a/buildstream/plugins/elements/compose.py
+++ b/buildstream/plugins/elements/compose.py
@@ -101,7 +101,6 @@ class ComposeElement(Element):
             f: getmtime(os.path.join(basedir, f))
             for f in utils.list_relative_paths(basedir)
         }
-        manifest = []
         integration_files = []
 
         # Run any integration commands provided by the dependencies
@@ -118,8 +117,6 @@ class ComposeElement(Element):
                 ]
                 self.info("Integration effected {} files".format(len(integration_files)))
 
-        manifest += integration_files
-
         # The remainder of this is expensive, make an early exit if
         # we're not being selective about what is to be included.
         if not (self.include or self.exclude) and self.include_orphans:
@@ -154,18 +151,24 @@ class ComposeElement(Element):
         detail = "\n".join(lines)
 
         with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
-            self.stage_dependency_artifacts(sandbox, Scope.BUILD,
-                                            path=stagedir,
-                                            include=self.include,
-                                            exclude=self.exclude,
-                                            orphans=self.include_orphans)
+            manifest = self.stage_dependency_artifacts(sandbox, Scope.BUILD,
+                                                       path=stagedir,
+                                                       include=self.include,
+                                                       exclude=self.exclude,
+                                                       orphans=self.include_orphans)
 
             if self.integration:
                 self.status("Moving {} integration files".format(len(integration_files)))
                 utils.move_files(basedir, installdir, integration_files)
 
+                for filename in integration_files:
+                    if filename in manifest:
+                        manifest[filename] = manifest[filename] + ('integration',)
+                    else:
+                        manifest[filename] = ('integration',)
+
         # And we're done
-        return os.path.join(os.sep, 'buildstream', 'install')
+        return (os.path.join(os.sep, 'buildstream', 'install'), manifest)
 
 
 # Like os.path.getmtime(), but doesnt explode on symlinks
diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py
index 3ab7f7833..7c3e2394f 100644
--- a/buildstream/plugins/elements/import.py
+++ b/buildstream/plugins/elements/import.py
@@ -89,7 +89,7 @@ class ImportElement(BuildElement):
         shutil.move(inputdir, outputdir)
 
         # And we're done
-        return '/output'
+        return '/output', {}
 
     def generate_script(self):
         build_root = self.get_variable('build-root')
diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py
index 45c49c514..593be7cbd 100644
--- a/buildstream/plugins/elements/stack.py
+++ b/buildstream/plugins/elements/stack.py
@@ -67,7 +67,7 @@ class StackElement(Element):
         os.makedirs(os.path.dirname(outputdir), exist_ok=True)
 
         # And we're done
-        return '/output'
+        return '/output', {}
 
 
 # Plugin entry point
diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py
index ce9bf95b5..c66953995 100644
--- a/buildstream/scriptelement.py
+++ b/buildstream/scriptelement.py
@@ -281,7 +281,7 @@ class ScriptElement(Element):
                         raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode))
 
         # Return where the result can be collected from
-        return self.__install_root
+        return self.__install_root, {}
 
 
 def setup():
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 637d583b0..dfb58b46d 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -60,6 +60,9 @@ class FileListResult():
         self.failed_attributes = []
         """List of files for which attributes could not be copied over"""
 
+        self.manifest = []
+        """Dict mapping each file to its artifact and split-rules domain"""
+
 
 def list_relative_paths(directory):
     """A generator for walking directory relative paths