Merge remote-tracking branch 'origin/baserock/richarddale/metadata_contents'

Conflicts:
	morphlib/app.py

Also fixed copyright year in tests/trove-id.script, which was not from
Richard's patch, but from an earlier commit, and all my own fault.

7 files changed, 65 insertions, 25 deletions

diff --git a/morphlib/artifact.py b/morphlib/artifact.py
index 82680709..20fdb185 100644
--- a/morphlib/artifact.py
+++ b/morphlib/artifact.py
@@ -26,6 +26,9 @@ class Artifact(object):
     * ``cache_id`` -- a dict describing the components of the cache key
     * ``dependencies`` -- list of Artifacts that need to be built beforehand
     * ``dependents`` -- list of Artifacts that need this Artifact to be built
+    * ``metadata_version`` -- When the format of the artifact metadata
+                              changes, this version number is raised causing 
+                              any existing cached artifacts to be invalidated
 
     The ``dependencies`` and ``dependents`` lists MUST be modified by
     the ``add_dependencies`` and ``add_dependent`` methods only.
@@ -39,6 +42,7 @@ class Artifact(object):
         self.cache_key = None
         self.dependencies = []
         self.dependents = []
+        self.metadata_version = 1
 
     def add_dependency(self, artifact):
         '''Add ``artifact`` to the dependency list.'''
diff --git a/morphlib/bins.py b/morphlib/bins.py
index 0374c117..6fb7dc5a 100644
--- a/morphlib/bins.py
+++ b/morphlib/bins.py
@@ -50,26 +50,19 @@ def safe_makefile(self, tarinfo, targetpath):
 tarfile.TarFile.makefile = safe_makefile
 
 
-def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
-    '''Create a chunk from the contents of a directory.
+def _chunk_filenames(rootdir, regexps, dump_memory_profile=None):
+
+    '''Return the filenames for a chunk from the contents of a directory.
 
     Only files and directories that match at least one of the regular
     expressions are accepted. The regular expressions are implicitly
     anchored to the beginning of the string, but not the end. The
     filenames are relative to rootdir.
 
-    ``f`` is an open file handle, to which the tar file is written.
-
     '''
 
     dump_memory_profile = dump_memory_profile or (lambda msg: None)
 
-    # This timestamp is used to normalize the mtime for every file in
-    # chunk artifact. This is useful to avoid problems from smallish
-    # clock skew. It needs to be recent enough, however, that GNU tar
-    # does not complain about an implausibly old timestamp.
-    normalized_timestamp = 683074800
-
     def matches(filename):
         return any(x.match(filename) for x in compiled)
 
@@ -79,10 +72,6 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
             filename = os.path.dirname(filename)
             yield filename
 
-    logging.debug('Creating chunk file %s from %s with regexps %s' %
-                  (getattr(f, 'name', 'UNNAMED'), rootdir, regexps))
-    dump_memory_profile('at beginning of create_chunk')
-
     compiled = [re.compile(x) for x in regexps]
     include = set()
     for dirname, subdirs, basenames in os.walk(rootdir):
@@ -93,13 +82,45 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
             if matches(os.path.relpath(filename, rootdir)):
                 for name in names_to_root(filename):
                     if name not in include:
-                        logging.debug('regexp match: %s' % name)
                         include.add(name)
             else:
                 logging.debug('regexp MISMATCH: %s' % filename)
     dump_memory_profile('after walking')
 
-    include = sorted(include)  # get dirs before contents
+    return sorted(include)  # get dirs before contents
+
+
+def chunk_contents(rootdir, regexps):
+    ''' Return the list of files in a chunk, with the rootdir
+        stripped off.
+    
+    '''
+    
+    filenames = _chunk_filenames(rootdir, regexps)
+    # The first entry is the rootdir directory, which we don't need
+    filenames.pop(0)
+    contents = [str[len(rootdir):] for str in filenames]
+    return contents
+
+
+def create_chunk(rootdir, f, regexps, dump_memory_profile=None):
+    '''Create a chunk from the contents of a directory.
+    
+    ``f`` is an open file handle, to which the tar file is written.
+
+    '''
+
+    dump_memory_profile = dump_memory_profile or (lambda msg: None)
+
+    # This timestamp is used to normalize the mtime for every file in
+    # chunk artifact. This is useful to avoid problems from smallish
+    # clock skew. It needs to be recent enough, however, that GNU tar
+    # does not complain about an implausibly old timestamp.
+    normalized_timestamp = 683074800
+
+    include = _chunk_filenames(rootdir, regexps, dump_memory_profile)
+    dump_memory_profile('at beginning of create_chunk')
+    
     tar = tarfile.open(fileobj=f, mode='w')
     for filename in include:
         # Normalize mtime for everything.
diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py
index edefb092..a9a94a44 100644
--- a/morphlib/bins_tests.py
+++ b/morphlib/bins_tests.py
@@ -112,6 +112,10 @@ class ChunkTests(BinsTest):
         morphlib.bins.create_chunk(self.instdir, self.chunk_f, regexps)
         self.chunk_f.flush()
 
+    def chunk_contents(self, regexps):
+        self.populate_instdir()
+        return morphlib.bins.chunk_contents(self.instdir, regexps)
+
     def unpack_chunk(self):
         os.mkdir(self.unpacked)
         morphlib.bins.unpack_binary(self.chunk_file, self.unpacked)
@@ -135,6 +139,11 @@ class ChunkTests(BinsTest):
         self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)],
                          ['.', 'lib', 'lib/libfoo.so'])
 
+    def test_list_chunk_contents(self):
+        contents = self.chunk_contents(['.'])
+        self.assertEqual(contents,
+                         ['/bin', '/bin/foo', '/lib', '/lib/libfoo.so'])
+
     def test_does_not_compress_artifact(self):
         self.create_chunk(['bin'])
         f = gzip.open(self.chunk_file)
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index b8026612..9aefabb6 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -186,7 +186,7 @@ class BuilderBase(object):
             json.dump(meta, f, indent=4, sort_keys=True)
             f.write('\n')
 
-    def create_metadata(self, artifact_name):
+    def create_metadata(self, artifact_name, contents=[]):
         '''Create metadata to artifact to allow it to be reproduced later.
 
         The metadata is represented as a dict, which later on will be
@@ -214,6 +214,7 @@ class BuilderBase(object):
                 'commit': morphlib.gitversion.commit,
                 'version': morphlib.gitversion.version,
             },
+            'contents': contents,
         }
 
         return meta
@@ -225,7 +226,7 @@ class BuilderBase(object):
             os.makedirs(dirname)
         return open(filename, mode)
 
-    def write_metadata(self, instdir, artifact_name):
+    def write_metadata(self, instdir, artifact_name, contents=[]):
         '''Write the metadata for an artifact.
 
         The file will be located under the ``baserock`` directory under
@@ -234,7 +235,7 @@ class BuilderBase(object):
 
         '''
 
-        meta = self.create_metadata(artifact_name)
+        meta = self.create_metadata(artifact_name, contents)
 
         basename = '%s.meta' % artifact_name
         filename = os.path.join(instdir, 'baserock', basename)
@@ -428,6 +429,7 @@ class ChunkBuilder(BuilderBase):
 
     def assemble_chunk_artifacts(self, destdir):  # pragma: no cover
         built_artifacts = []
+        filenames = []
         with self.build_watch('create-chunks'):
             specs = self.artifact.source.morphology['chunks']
             if len(specs) == 0:
@@ -437,12 +439,14 @@ class ChunkBuilder(BuilderBase):
             names = specs.keys()
             names.sort(key=lambda name: [ord(c) for c in name])
             for artifact_name in names:
-                self.write_metadata(destdir, artifact_name)
+                artifact = self.new_artifact(artifact_name)
                 patterns = specs[artifact_name]
                 patterns += [r'baserock/%s\.' % artifact_name]
 
-                artifact = self.new_artifact(artifact_name)
                 with self.local_artifact_cache.put(artifact) as f:
+                    contents = morphlib.bins.chunk_contents(destdir, patterns)
+                    self.write_metadata(destdir, artifact_name, contents)
+
                     logging.debug('assembling chunk %s' % artifact_name)
                     logging.debug('assembling into %s' % f.name)
                     self.app.status(msg='Creating chunk artifact %(name)s',
@@ -498,7 +502,8 @@ class StratumBuilder(BuilderBase):
                 lac = self.local_artifact_cache
                 artifact_name = self.artifact.source.morphology['name']
                 artifact = self.new_artifact(artifact_name)
-                meta = self.create_metadata(artifact_name)
+                contents = [x.name for x in constituents]
+                meta = self.create_metadata(artifact_name, contents)
                 with lac.put_artifact_metadata(artifact, 'meta') as f:
                     json.dump(meta, f, indent=4, sort_keys=True)
                 with self.local_artifact_cache.put(artifact) as f:
diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py
index 6acf654b..d7e2e3b1 100644
--- a/morphlib/cachekeycomputer.py
+++ b/morphlib/cachekeycomputer.py
@@ -81,7 +81,8 @@ class CacheKeyComputer(object):
         keys = {
             'env': self._filterenv(self._build_env.env),
             'filename': artifact.source.filename,
-            'kids': [self.compute_key(x) for x in artifact.dependencies]
+            'kids': [self.compute_key(x) for x in artifact.dependencies],
+            'metadata-version': artifact.metadata_version
         }
 
         kind = artifact.source.morphology['kind']
diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stderr b/tests.as-root/run-in-artifact-with-different-artifacts.stderr
index b37e8f88..5fc4fb0f 100644
--- a/tests.as-root/run-in-artifact-with-different-artifacts.stderr
+++ b/tests.as-root/run-in-artifact-with-different-artifacts.stderr
@@ -1 +1 @@
-ERROR: Artifact TMP/cache/artifacts/cb886f5b5aa3c4f4a36b5f763b8330077b38681573a1edcbed3554aef0489b37.stratum.linux-stratum cannot be extracted or mounted
+ERROR: Artifact TMP/cache/artifacts/67596ea97123eeb66afce92675dbb63eb8fd840d01f38902d4bf6f573b609499.stratum.linux-stratum cannot be extracted or mounted
diff --git a/tests/trove-id.script b/tests/trove-id.script
index 7157559b..998bde44 100755
--- a/tests/trove-id.script
+++ b/tests/trove-id.script
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by