builder: Use the OSTree artifact cache when building

The API of the OSTree artifact cache is slightly different to that of the
old tarball cache, so adjust things accordingly. Also, only store the
files changed at system-construction-time rather than everything in
system artifacts.

3 files changed, 49 insertions, 45 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index d7c445df..c4d7a6f5 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -422,8 +422,10 @@ class BuildCommand(object):
             # module into morphlib.remoteartififactcache first.
             to_fetch = []
             if not self.lac.has(artifact):
-                to_fetch.append((self.rac.get(artifact),
-                                 self.lac.put(artifact)))
+                self.app.status(
+                    msg='Fetching to local cache: artifact %(name)s',
+                    name=artifact.name)
+                self.lac.copy_from_remote(artifact, self.rac)
 
             if artifact.source.morphology.needs_artifact_metadata_cached:
                 if not self.lac.has_artifact_metadata(artifact, 'meta'):
@@ -432,9 +434,6 @@ class BuildCommand(object):
                         self.lac.put_artifact_metadata(artifact, 'meta')))
 
             if len(to_fetch) > 0:
-                self.app.status(
-                    msg='Fetching to local cache: artifact %(name)s',
-                    name=artifact.name)
                 fetch_files(to_fetch)
 
     def create_staging_area(self, build_env, use_chroot=True, extra_env={},
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 0c681353..9b01f983 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -432,13 +432,23 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                with self.local_artifact_cache.put(chunk_artifact) as f:
-                    self.write_metadata(destdir, chunk_artifact_name,
-                                        parented_paths)
-
-                    self.app.status(msg='Creating chunk artifact %(name)s',
-                                    name=chunk_artifact_name)
-                    morphlib.bins.create_chunk(destdir, f, parented_paths)
+                self.write_metadata(destdir, chunk_artifact_name,
+                                    parented_paths)
+
+                self.app.status(msg='Creating chunk artifact %(name)s',
+                                name=chunk_artifact_name)
+                # TODO: This is not concurrency safe, bins.create_chunk will
+                #       fail if tempdir already exists (eg if another build
+                #       has created it).
+                tempdir = os.path.join(self.app.settings['tempdir'],
+                                       chunk_artifact.basename())
+                try:
+                    morphlib.bins.create_chunk(destdir, tempdir,
+                                               parented_paths)
+                    self.local_artifact_cache.put(tempdir, chunk_artifact)
+                finally:
+                    if os.path.isdir(tempdir):
+                        shutil.rmtree(tempdir)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -482,8 +492,13 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    with self.local_artifact_cache.put(a) as f:
+                    # TODO: This is not concurrency safe, put_stratum_artifact
+                    #       deletes temp which could be in use by another
+                    #       build.
+                    temp = os.path.join(self.app.settings['tempdir'], a.name)
+                    with open(temp, 'w+') as f:
                         json.dump([c.basename() for c in constituents], f)
+                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -505,7 +520,6 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
-                handle = self.local_artifact_cache.put(artifact)
                 try:
                     fs_root = self.staging_area.destdir(self.source)
                     self.unpack_strata(fs_root)
@@ -524,18 +538,10 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                                                    union_filesystem)
                     self.write_metadata(editable_root, a_name)
                     self.run_system_integration_commands(editable_root)
-                    unslashy_root = editable_root[1:]
-                    def uproot_info(info):
-                        info.name = relpath(info.name, unslashy_root)
-                        if info.islnk():
-                            info.linkname = relpath(info.linkname,
-                                                    unslashy_root)
-                        return info
-                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
-                    self.app.status(msg='Constructing tarball of rootfs',
-                                    chatty=True)
-                    tar.add(editable_root, recursive=True, filter=uproot_info)
-                    tar.close()
+                    # Put the contents of upperdir into the local artifact
+                    # cache. Don't use editable root as we only want to
+                    # store the modified files.
+                    self.local_artifact_cache.put(upperdir, artifact)
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
@@ -543,13 +549,11 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.abort()
                     raise
                 else:
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.close()
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
@@ -558,13 +562,12 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
         '''Unpack a single stratum into a target directory'''
 
         cache = self.local_artifact_cache
-        with cache.get(stratum_artifact) as stratum_file:
+        with open(cache.get(stratum_artifact), 'r') as stratum_file:
             artifact_list = json.load(stratum_file, encoding='unicode-escape')
             for chunk in (ArtifactCacheReference(a) for a in artifact_list):
-                self.app.status(msg='Unpacking chunk %(basename)s',
+                self.app.status(msg='Checkout chunk %(basename)s',
                                 basename=chunk.basename(), chatty=True)
-                with cache.get(chunk) as chunk_file:
-                    morphlib.bins.unpack_binary_from_file(chunk_file, target)
+                cache.get(chunk, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -575,7 +578,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Unpacking strata to %(path)s',
+        self.app.status(msg='Checking out strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():
@@ -587,12 +590,14 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
                 # download the chunk artifacts if necessary
                 for stratum_artifact in self.source.dependencies:
-                    f = self.local_artifact_cache.get(stratum_artifact)
-                    chunks = [ArtifactCacheReference(c) for c in json.load(f)]
+                    stratum_path = self.local_artifact_cache.get(
+                        stratum_artifact)
+                    with open(stratum_path, 'r') as stratum:
+                        chunks = [ArtifactCacheReference(c)
+                                  for c in json.load(stratum)]
                     download_depends(chunks,
                                      self.local_artifact_cache,
                                      self.remote_artifact_cache)
-                    f.close()
 
                 # unpack it from the local artifact cache
                 for stratum_artifact in self.source.dependencies:
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index a571e3d0..b5e66521 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -105,8 +105,8 @@ class FakeArtifactCache(object):
     def __init__(self):
         self._cached = {}
 
-    def put(self, artifact):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name))
+    def put(self, artifact, directory):
+        self._cached[(artifact.cache_key, artifact.name)] = artifact.name
 
     def put_artifact_metadata(self, artifact, name):
         return FakeFileHandle(self, (artifact.cache_key, artifact.name, name))
@@ -114,7 +114,7 @@ class FakeArtifactCache(object):
     def put_source_metadata(self, source, cachekey, name):
         return FakeFileHandle(self, (cachekey, name))
 
-    def get(self, artifact):
+    def get(self, artifact, directory=None):
         return StringIO.StringIO(
             self._cached[(artifact.cache_key, artifact.name)])
 
@@ -134,6 +134,10 @@ class FakeArtifactCache(object):
     def has_source_metadata(self, source, cachekey, name):
         return (cachekey, name) in self._cached
 
+    def copy_from_remote(self, artifact, remote):
+        self._cached[(artifact.cache_key, artifact.name)] = \
+            remote._cached[(artifact.cache_key, artifact.name)]
+
 
 class BuilderBaseTests(unittest.TestCase):
 
@@ -191,9 +195,7 @@ class BuilderBaseTests(unittest.TestCase):
         rac = FakeArtifactCache()
         afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
         for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
+            rac.put(a, 'not-a-dir')
         morphlib.builder.download_depends(afacts, lac, rac)
         self.assertTrue(all(lac.has(a) for a in afacts))
 
@@ -202,9 +204,7 @@ class BuilderBaseTests(unittest.TestCase):
         rac = FakeArtifactCache()
         afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
         for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
+            rac.put(a, 'not-a-dir')
             fh = rac.put_artifact_metadata(a, 'meta')
             fh.write('metadata')
             fh.close()