builder: Use the OSTree artifact cache when building

The API of the OSTree artifact cache is slightly different to that of the
old tarball cache, so adjust things accordingly. Also, only store the
files changed at system-construction-time rather than everything in
system artifacts.

Change-Id: Iec74f37ed1ae769280c552a1d67f5f6c5fdae9e3

3 files changed, 45 insertions, 43 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 180aa2a6..76f8bd99 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -423,8 +423,10 @@ class BuildCommand(object):
             # module into morphlib.remoteartififactcache first.
             to_fetch = []
             if not self.lac.has(artifact):
-                to_fetch.append((self.rac.get(artifact),
-                                 self.lac.put(artifact)))
+                self.app.status(
+                    msg='Fetching to local cache: artifact %(name)s',
+                    name=artifact.name)
+                self.lac.copy_from_remote(artifact, self.rac)
 
             if artifact.source.morphology.needs_artifact_metadata_cached:
                 if not self.lac.has_artifact_metadata(artifact, 'meta'):
@@ -433,9 +435,6 @@ class BuildCommand(object):
                         self.lac.put_artifact_metadata(artifact, 'meta')))
 
             if len(to_fetch) > 0:
-                self.app.status(
-                    msg='Fetching to local cache: artifact %(name)s',
-                    name=artifact.name)
                 fetch_files(to_fetch)
 
     def create_staging_area(self, build_env, use_chroot=True, extra_env={},
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 6fc871fc..b0c95bb3 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -432,13 +432,23 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                with self.local_artifact_cache.put(chunk_artifact) as f:
-                    self.write_metadata(destdir, chunk_artifact_name,
-                                        parented_paths)
-
-                    self.app.status(msg='Creating chunk artifact %(name)s',
-                                    name=chunk_artifact_name)
-                    morphlib.bins.create_chunk(destdir, f, parented_paths)
+                self.write_metadata(destdir, chunk_artifact_name,
+                                    parented_paths)
+
+                self.app.status(msg='Creating chunk artifact %(name)s',
+                                name=chunk_artifact_name)
+                # TODO: This is not concurrency safe, bins.create_chunk will
+                #       fail if tempdir already exists (eg if another build
+                #       has created it).
+                tempdir = os.path.join(self.app.settings['tempdir'],
+                                       chunk_artifact.basename())
+                try:
+                    morphlib.bins.create_chunk(destdir, tempdir,
+                                               parented_paths)
+                    self.local_artifact_cache.put(tempdir, chunk_artifact)
+                finally:
+                    if os.path.isdir(tempdir):
+                        shutil.rmtree(tempdir)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -482,8 +492,13 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    with self.local_artifact_cache.put(a) as f:
+                    # TODO: This is not concurrency safe, put_stratum_artifact
+                    #       deletes temp which could be in use by another
+                    #       build.
+                    temp = os.path.join(self.app.settings['tempdir'], a.name)
+                    with open(temp, 'w+') as f:
                         json.dump([c.basename() for c in constituents], f)
+                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -505,10 +520,8 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
-                handle = self.local_artifact_cache.put(artifact)
                 try:
                     fs_root = self.staging_area.destdir(self.source)
-                    self.unpack_strata(fs_root)
                     upperdir = self.staging_area.overlay_upperdir(
                         self.source)
                     editable_root = self.staging_area.overlaydir(self.source)
@@ -522,20 +535,13 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                                                    editable_root, fs_root,
                                                    upperdir, workdir,
                                                    union_filesystem)
+                    self.unpack_strata(fs_root)
                     self.write_metadata(editable_root, a_name)
                     self.run_system_integration_commands(editable_root)
-                    unslashy_root = editable_root[1:]
-                    def uproot_info(info):
-                        info.name = relpath(info.name, unslashy_root)
-                        if info.islnk():
-                            info.linkname = relpath(info.linkname,
-                                                    unslashy_root)
-                        return info
-                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
-                    self.app.status(msg='Constructing tarball of rootfs',
-                                    chatty=True)
-                    tar.add(editable_root, recursive=True, filter=uproot_info)
-                    tar.close()
+                    # Put the contents of upperdir into the local artifact
+                    # cache. Don't use editable root as we only want to
+                    # store the modified files.
+                    self.local_artifact_cache.put(upperdir, artifact)
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
@@ -543,13 +549,11 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.abort()
                     raise
                 else:
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.close()
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
@@ -562,7 +566,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
         '''
         cache = self.local_artifact_cache
-        with cache.get(stratum_artifact) as stratum_file:
+        with open(cache.get(stratum_artifact), 'r') as stratum_file:
             try:
                 artifact_list = json.load(stratum_file,
                                           encoding='unicode-escape')
@@ -577,10 +581,9 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
         cache = self.local_artifact_cache
         for chunk in self.load_stratum(stratum_artifact):
-            self.app.status(msg='Unpacking chunk %(basename)s',
+            self.app.status(msg='Checkout chunk %(basename)s',
                             basename=chunk.basename(), chatty=True)
-            with cache.get(chunk) as chunk_file:
-                morphlib.bins.unpack_binary_from_file(chunk_file, target)
+            cache.get(chunk, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -591,7 +594,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Unpacking strata to %(path)s',
+        self.app.status(msg='Checking out strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index a571e3d0..b5e66521 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -105,8 +105,8 @@ class FakeArtifactCache(object):
     def __init__(self):
         self._cached = {}
 
-    def put(self, artifact):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name))
+    def put(self, artifact, directory):
+        self._cached[(artifact.cache_key, artifact.name)] = artifact.name
 
     def put_artifact_metadata(self, artifact, name):
         return FakeFileHandle(self, (artifact.cache_key, artifact.name, name))
@@ -114,7 +114,7 @@ class FakeArtifactCache(object):
     def put_source_metadata(self, source, cachekey, name):
         return FakeFileHandle(self, (cachekey, name))
 
-    def get(self, artifact):
+    def get(self, artifact, directory=None):
         return StringIO.StringIO(
             self._cached[(artifact.cache_key, artifact.name)])
 
@@ -134,6 +134,10 @@ class FakeArtifactCache(object):
     def has_source_metadata(self, source, cachekey, name):
         return (cachekey, name) in self._cached
 
+    def copy_from_remote(self, artifact, remote):
+        self._cached[(artifact.cache_key, artifact.name)] = \
+            remote._cached[(artifact.cache_key, artifact.name)]
+
 
 class BuilderBaseTests(unittest.TestCase):
 
@@ -191,9 +195,7 @@ class BuilderBaseTests(unittest.TestCase):
         rac = FakeArtifactCache()
         afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
         for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
+            rac.put(a, 'not-a-dir')
         morphlib.builder.download_depends(afacts, lac, rac)
         self.assertTrue(all(lac.has(a) for a in afacts))
 
@@ -202,9 +204,7 @@ class BuilderBaseTests(unittest.TestCase):
         rac = FakeArtifactCache()
         afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
         for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
+            rac.put(a, 'not-a-dir')
             fh = rac.put_artifact_metadata(a, 'meta')
             fh.write('metadata')
             fh.close()