builder: Use the OSTree artifact cache when building

The API of the OSTree artifact cache is slightly different to that of the
old tarball cache, so adjust things accordingly. Also, only store the
files changed at system-construction-time rather than everything in
system artifacts.

3 files changed, 49 insertions, 45 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 38057365..a6ce5e91 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -419,8 +419,10 @@ class BuildCommand(object):
             # module into morphlib.remoteartififactcache first.
             to_fetch = []
             if not self.lac.has(artifact):
-                to_fetch.append((self.rac.get(artifact),
-                                 self.lac.put(artifact)))
+                self.app.status(
+                    msg='Fetching to local cache: artifact %(name)s',
+                    name=artifact.name)
+                self.lac.copy_from_remote(artifact, self.rac)
 
             if artifact.source.morphology.needs_artifact_metadata_cached:
                 if not self.lac.has_artifact_metadata(artifact, 'meta'):
@@ -429,9 +431,6 @@ class BuildCommand(object):
                         self.lac.put_artifact_metadata(artifact, 'meta')))
 
             if len(to_fetch) > 0:
-                self.app.status(
-                    msg='Fetching to local cache: artifact %(name)s',
-                    name=artifact.name)
                 fetch_files(to_fetch)
 
     def create_staging_area(self, build_env, use_chroot=True, extra_env={},
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 9991bce2..8c28b7c5 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -433,13 +433,23 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                with self.local_artifact_cache.put(chunk_artifact) as f:
-                    self.write_metadata(destdir, chunk_artifact_name,
-                                        parented_paths)
-
-                    self.app.status(msg='Creating chunk artifact %(name)s',
-                                    name=chunk_artifact_name)
-                    morphlib.bins.create_chunk(destdir, f, parented_paths)
+                self.write_metadata(destdir, chunk_artifact_name,
+                                    parented_paths)
+
+                self.app.status(msg='Creating chunk artifact %(name)s',
+                                name=chunk_artifact_name)
+                # TODO: This is not concurrency safe, bins.create_chunk will
+                #       fail if tempdir already exists (eg if another build
+                #       has created it).
+                tempdir = os.path.join(self.app.settings['tempdir'],
+                                       chunk_artifact.basename())
+                try:
+                    morphlib.bins.create_chunk(destdir, tempdir,
+                                               parented_paths)
+                    self.local_artifact_cache.put(tempdir, chunk_artifact)
+                finally:
+                    if os.path.isdir(tempdir):
+                        shutil.rmtree(tempdir)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -483,8 +493,13 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    with self.local_artifact_cache.put(a) as f:
+                    # TODO: This is not concurrency safe, put_stratum_artifact
+                    #       deletes temp which could be in use by another
+                    #       build.
+                    temp = os.path.join(self.app.settings['tempdir'], a.name)
+                    with open(temp, 'w+') as f:
                         json.dump([c.basename() for c in constituents], f)
+                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -506,7 +521,6 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
-                handle = self.local_artifact_cache.put(artifact)
                 try:
                     fs_root = self.staging_area.destdir(self.source)
                     self.unpack_strata(fs_root)
@@ -525,18 +539,10 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                                                    union_filesystem)
                     self.write_metadata(editable_root, a_name)
                     self.run_system_integration_commands(editable_root)
-                    unslashy_root = editable_root[1:]
-                    def uproot_info(info):
-                        info.name = relpath(info.name, unslashy_root)
-                        if info.islnk():
-                            info.linkname = relpath(info.linkname,
-                                                    unslashy_root)
-                        return info
-                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
-                    self.app.status(msg='Constructing tarball of rootfs',
-                                    chatty=True)
-                    tar.add(editable_root, recursive=True, filter=uproot_info)
-                    tar.close()
+                    # Put the contents of upperdir into the local artifact
+                    # cache. Don't use editable root as we only want to
+                    # store the modified files.
+                    self.local_artifact_cache.put(upperdir, artifact)
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
@@ -544,13 +550,11 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.abort()
                     raise
                 else:
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.close()
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
@@ -559,13 +563,12 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
         '''Unpack a single stratum into a target directory'''
 
         cache = self.local_artifact_cache
-        with cache.get(stratum_artifact) as stratum_file:
+        with open(cache.get(stratum_artifact), 'r') as stratum_file:
             artifact_list = json.load(stratum_file, encoding='unicode-escape')
             for chunk in (ArtifactCacheReference(a) for a in artifact_list):
-                self.app.status(msg='Unpacking chunk %(basename)s',
+                self.app.status(msg='Checkout chunk %(basename)s',
                                 basename=chunk.basename(), chatty=True)
-                with cache.get(chunk) as chunk_file:
-                    morphlib.bins.unpack_binary_from_file(chunk_file, target)
+                cache.get(chunk, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -576,7 +579,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Unpacking strata to %(path)s',
+        self.app.status(msg='Checking out strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():
@@ -588,12 +591,14 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
                 # download the chunk artifacts if necessary
                 for stratum_artifact in self.source.dependencies:
-                    f = self.local_artifact_cache.get(stratum_artifact)
-                    chunks = [ArtifactCacheReference(c) for c in json.load(f)]
+                    stratum_path = self.local_artifact_cache.get(
+                        stratum_artifact)
+                    with open(stratum_path, 'r') as stratum:
+                        chunks = [ArtifactCacheReference(c)
+                                  for c in json.load(stratum)]
                     download_depends(chunks,
                                      self.local_artifact_cache,
                                      self.remote_artifact_cache)
-                    f.close()
 
                 # unpack it from the local artifact cache
                 for stratum_artifact in self.source.dependencies:
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 0cc90819..54040608 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -106,8 +106,8 @@ class FakeArtifactCache(object):
     def __init__(self):
         self._cached = {}
 
-    def put(self, artifact):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name))
+    def put(self, artifact, directory):
+        self._cached[(artifact.cache_key, artifact.name)] = artifact.name
 
     def put_artifact_metadata(self, artifact, name):
         return FakeFileHandle(self, (artifact.cache_key, artifact.name, name))
@@ -115,7 +115,7 @@ class FakeArtifactCache(object):
     def put_source_metadata(self, source, cachekey, name):
         return FakeFileHandle(self, (cachekey, name))
 
-    def get(self, artifact):
+    def get(self, artifact, directory=None):
         return StringIO.StringIO(
             self._cached[(artifact.cache_key, artifact.name)])
 
@@ -135,6 +135,10 @@ class FakeArtifactCache(object):
     def has_source_metadata(self, source, cachekey, name):
         return (cachekey, name) in self._cached
 
+    def copy_from_remote(self, artifact, remote):
+        self._cached[(artifact.cache_key, artifact.name)] = \
+            remote._cached[(artifact.cache_key, artifact.name)]
+
 
 class BuilderBaseTests(unittest.TestCase):
 
@@ -192,9 +196,7 @@ class BuilderBaseTests(unittest.TestCase):
         rac = FakeArtifactCache()
         afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
         for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
+            rac.put(a, 'not-a-dir')
         morphlib.builder.download_depends(afacts, lac, rac)
         self.assertTrue(all(lac.has(a) for a in afacts))
 
@@ -203,9 +205,7 @@ class BuilderBaseTests(unittest.TestCase):
         rac = FakeArtifactCache()
         afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
         for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
+            rac.put(a, 'not-a-dir')
             fh = rac.put_artifact_metadata(a, 'meta')
             fh.write('metadata')
             fh.close()