Add contents list to chunk and stratum metadata

3 files changed, 55 insertions, 21 deletions

diff --git a/morphlib/artifact.py b/morphlib/artifact.py
index 82680709..2f9c65df 100644
--- a/morphlib/artifact.py
+++ b/morphlib/artifact.py
@@ -24,6 +24,7 @@ class Artifact(object):
     * ``name`` -- the name of the artifact
     * ``cache_key`` -- a cache key to uniquely identify the artifact
     * ``cache_id`` -- a dict describing the components of the cache key
+    * ``contents`` -- a list of the installed files or chunks in the artifact
     * ``dependencies`` -- list of Artifacts that need to be built beforehand
     * ``dependents`` -- list of Artifacts that need this Artifact to be built
 
@@ -39,6 +40,7 @@ class Artifact(object):
         self.cache_key = None
         self.dependencies = []
         self.dependents = []
+        self.contents = []
 
     def add_dependency(self, artifact):
         '''Add ``artifact`` to the dependency list.'''
diff --git a/morphlib/bins.py b/morphlib/bins.py
index 0374c117..1721c69a 100644
--- a/morphlib/bins.py
+++ b/morphlib/bins.py
@@ -50,26 +50,19 @@ def safe_makefile(self, tarinfo, targetpath):
 tarfile.TarFile.makefile = safe_makefile
 
 
-def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
-    '''Create a chunk from the contents of a directory.
+def chunk_filenames(rootdir, regexps, dump_memory_profile=None):
+
+    '''Return the filenames for a chunk from the contents of a directory.
 
     Only files and directories that match at least one of the regular
     expressions are accepted. The regular expressions are implicitly
     anchored to the beginning of the string, but not the end. The
     filenames are relative to rootdir.
 
-    ``f`` is an open file handle, to which the tar file is written.
-
     '''
 
     dump_memory_profile = dump_memory_profile or (lambda msg: None)
 
-    # This timestamp is used to normalize the mtime for every file in
-    # chunk artifact. This is useful to avoid problems from smallish
-    # clock skew. It needs to be recent enough, however, that GNU tar
-    # does not complain about an implausibly old timestamp.
-    normalized_timestamp = 683074800
-
     def matches(filename):
         return any(x.match(filename) for x in compiled)
 
@@ -79,10 +72,7 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
             filename = os.path.dirname(filename)
             yield filename
 
-    logging.debug('Creating chunk file %s from %s with regexps %s' %
-                  (getattr(f, 'name', 'UNNAMED'), rootdir, regexps))
-    dump_memory_profile('at beginning of create_chunk')
-
+    logging.debug('regexps: %s' % repr(regexps))
     compiled = [re.compile(x) for x in regexps]
     include = set()
     for dirname, subdirs, basenames in os.walk(rootdir):
@@ -99,7 +89,42 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
                 logging.debug('regexp MISMATCH: %s' % filename)
     dump_memory_profile('after walking')
 
-    include = sorted(include)  # get dirs before contents
+    return sorted(include)  # get dirs before contents
+
+
+def chunk_contents(rootdir, regexps):
+    ''' Return the list of files in a chunk, with the rootdir
+        stripped off.
+    
+    '''
+    
+    filenames = chunk_filenames(rootdir, regexps)
+    # The first entry is the rootdir directory, which we don't need
+    filenames.pop(0)
+    contents = [str[len(rootdir):] for str in filenames]
+    return contents
+
+
+def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
+    '''Create a chunk from the contents of a directory.
+    
+    ``f`` is an open file handle, to which the tar file is written.
+
+    '''
+
+    dump_memory_profile = dump_memory_profile or (lambda msg: None)
+
+    # This timestamp is used to normalize the mtime for every file in
+    # chunk artifact. This is useful to avoid problems from smallish
+    # clock skew. It needs to be recent enough, however, that GNU tar
+    # does not complain about an implausibly old timestamp.
+    normalized_timestamp = 683074800
+
+    include = chunk_filenames(rootdir, regexps, dump_memory_profile)
+    logging.debug('Creating chunk file %s from %s with regexps %s' %
+                  (getattr(f, 'name', 'UNNAMED'), rootdir, regexps))
+    dump_memory_profile('at beginning of create_chunk')
+    
     tar = tarfile.open(fileobj=f, mode='w')
     for filename in include:
         # Normalize mtime for everything.
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index b8026612..cff74a8e 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -186,7 +186,7 @@ class BuilderBase(object):
             json.dump(meta, f, indent=4, sort_keys=True)
             f.write('\n')
 
-    def create_metadata(self, artifact_name):
+    def create_metadata(self, artifact_name, contents = []):
         '''Create metadata to artifact to allow it to be reproduced later.
 
         The metadata is represented as a dict, which later on will be
@@ -214,6 +214,8 @@ class BuilderBase(object):
                 'commit': morphlib.gitversion.commit,
                 'version': morphlib.gitversion.version,
             },
+            'contents': contents,
+            'metadata-version': 1,
         }
 
         return meta
@@ -225,7 +227,7 @@ class BuilderBase(object):
             os.makedirs(dirname)
         return open(filename, mode)
 
-    def write_metadata(self, instdir, artifact_name):
+    def write_metadata(self, instdir, artifact_name, contents = []):
         '''Write the metadata for an artifact.
 
         The file will be located under the ``baserock`` directory under
@@ -234,7 +236,7 @@ class BuilderBase(object):
 
         '''
 
-        meta = self.create_metadata(artifact_name)
+        meta = self.create_metadata(artifact_name, contents)
 
         basename = '%s.meta' % artifact_name
         filename = os.path.join(instdir, 'baserock', basename)
@@ -428,6 +430,7 @@ class ChunkBuilder(BuilderBase):
 
     def assemble_chunk_artifacts(self, destdir):  # pragma: no cover
         built_artifacts = []
+        filenames = []
         with self.build_watch('create-chunks'):
             specs = self.artifact.source.morphology['chunks']
             if len(specs) == 0:
@@ -437,12 +440,15 @@ class ChunkBuilder(BuilderBase):
             names = specs.keys()
             names.sort(key=lambda name: [ord(c) for c in name])
             for artifact_name in names:
-                self.write_metadata(destdir, artifact_name)
+                artifact = self.new_artifact(artifact_name)
                 patterns = specs[artifact_name]
                 patterns += [r'baserock/%s\.' % artifact_name]
 
-                artifact = self.new_artifact(artifact_name)
                 with self.local_artifact_cache.put(artifact) as f:
+                    contents = morphlib.bins.chunk_contents(destdir, patterns)
+                    logging.debug('metadata contents: %s' % contents)
+                    self.write_metadata(destdir, artifact_name, contents)
+
                     logging.debug('assembling chunk %s' % artifact_name)
                     logging.debug('assembling into %s' % f.name)
                     self.app.status(msg='Creating chunk artifact %(name)s',
@@ -498,7 +504,8 @@ class StratumBuilder(BuilderBase):
                 lac = self.local_artifact_cache
                 artifact_name = self.artifact.source.morphology['name']
                 artifact = self.new_artifact(artifact_name)
-                meta = self.create_metadata(artifact_name)
+                contents = [x.name for x in constituents]
+                meta = self.create_metadata(artifact_name, contents)
                 with lac.put_artifact_metadata(artifact, 'meta') as f:
                     json.dump(meta, f, indent=4, sort_keys=True)
                 with self.local_artifact_cache.put(artifact) as f: