builder: Use the OSTree artifact cache when building

The API of the OSTree artifact cache is slightly different to that of the
old tarball cache, so adjust things accordingly. Also, only store the
files changed at system-construction-time rather than everything in
system artifacts.

Change-Id: Iec74f37ed1ae769280c552a1d67f5f6c5fdae9e3

1 files changed, 32 insertions, 29 deletions

diff --git a/morphlib/builder.py b/morphlib/builder.py
index 6fc871fc..b0c95bb3 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -432,13 +432,23 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                with self.local_artifact_cache.put(chunk_artifact) as f:
-                    self.write_metadata(destdir, chunk_artifact_name,
-                                        parented_paths)
-
-                    self.app.status(msg='Creating chunk artifact %(name)s',
-                                    name=chunk_artifact_name)
-                    morphlib.bins.create_chunk(destdir, f, parented_paths)
+                self.write_metadata(destdir, chunk_artifact_name,
+                                    parented_paths)
+
+                self.app.status(msg='Creating chunk artifact %(name)s',
+                                name=chunk_artifact_name)
+                # TODO: This is not concurrency safe, bins.create_chunk will
+                #       fail if tempdir already exists (eg if another build
+                #       has created it).
+                tempdir = os.path.join(self.app.settings['tempdir'],
+                                       chunk_artifact.basename())
+                try:
+                    morphlib.bins.create_chunk(destdir, tempdir,
+                                               parented_paths)
+                    self.local_artifact_cache.put(tempdir, chunk_artifact)
+                finally:
+                    if os.path.isdir(tempdir):
+                        shutil.rmtree(tempdir)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -482,8 +492,13 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    with self.local_artifact_cache.put(a) as f:
+                    # TODO: This is not concurrency safe, put_stratum_artifact
+                    #       deletes temp which could be in use by another
+                    #       build.
+                    temp = os.path.join(self.app.settings['tempdir'], a.name)
+                    with open(temp, 'w+') as f:
                         json.dump([c.basename() for c in constituents], f)
+                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -505,10 +520,8 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
-                handle = self.local_artifact_cache.put(artifact)
                 try:
                     fs_root = self.staging_area.destdir(self.source)
-                    self.unpack_strata(fs_root)
                     upperdir = self.staging_area.overlay_upperdir(
                         self.source)
                     editable_root = self.staging_area.overlaydir(self.source)
@@ -522,20 +535,13 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                                                    editable_root, fs_root,
                                                    upperdir, workdir,
                                                    union_filesystem)
+                    self.unpack_strata(fs_root)
                     self.write_metadata(editable_root, a_name)
                     self.run_system_integration_commands(editable_root)
-                    unslashy_root = editable_root[1:]
-                    def uproot_info(info):
-                        info.name = relpath(info.name, unslashy_root)
-                        if info.islnk():
-                            info.linkname = relpath(info.linkname,
-                                                    unslashy_root)
-                        return info
-                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
-                    self.app.status(msg='Constructing tarball of rootfs',
-                                    chatty=True)
-                    tar.add(editable_root, recursive=True, filter=uproot_info)
-                    tar.close()
+                    # Put the contents of upperdir into the local artifact
+                    # cache. Don't use editable root as we only want to
+                    # store the modified files.
+                    self.local_artifact_cache.put(upperdir, artifact)
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
@@ -543,13 +549,11 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.abort()
                     raise
                 else:
                     if editable_root and os.path.exists(editable_root):
                         morphlib.fsutils.unmount(self.app.runcmd,
                                                  editable_root)
-                    handle.close()
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
@@ -562,7 +566,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
         '''
         cache = self.local_artifact_cache
-        with cache.get(stratum_artifact) as stratum_file:
+        with open(cache.get(stratum_artifact), 'r') as stratum_file:
             try:
                 artifact_list = json.load(stratum_file,
                                           encoding='unicode-escape')
@@ -577,10 +581,9 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
         cache = self.local_artifact_cache
         for chunk in self.load_stratum(stratum_artifact):
-            self.app.status(msg='Unpacking chunk %(basename)s',
+            self.app.status(msg='Checkout chunk %(basename)s',
                             basename=chunk.basename(), chatty=True)
-            with cache.get(chunk) as chunk_file:
-                morphlib.bins.unpack_binary_from_file(chunk_file, target)
+            cache.get(chunk, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -591,7 +594,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Unpacking strata to %(path)s',
+        self.app.status(msg='Checking out strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():