Use OSTree for hardlink and artifact cache and a CoW unionfs to make system artifacts fasterbaserock/richardmaw/ostree-squash

This replaces the artifact cache and the hardlink cache with an OSTree
repository, which is a great performance improvement when the cache
directory and temporary directory are on the same filesystem.

Additionally it can de-duplicate file contents.

When we construct system artifacts deploy them, the staging area needs
to be writable, so OSTree on its own is insufficient, as its hardlinks
require the root to be kept read-only.

To handle this we use either the in-kernel overlayfs or unionfs-fuse,
though there is no automatic fall-back and it needs to be specified
manually.

To support distributed building, the artifact cache is extended to
support an OSTree repository.

Unfortunately cross-bootstrap is not expected to work with these changes
at this point in time.

IMPORTANT NOTE: We are well aware that this patch is too large to be
comprehensible. We intend to revert and apply a cleaned up series when
it is ready.

Change-Id: I693bb752500dab3c6db3b97393689239ae7071a8

1 files changed, 51 insertions, 80 deletions

diff --git a/morphlib/builder.py b/morphlib/builder.py
index 1c016674..426c0ed0 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -28,7 +28,6 @@ import tempfile
 import cliapp
 
 import morphlib
-from morphlib.artifactcachereference import ArtifactCacheReference
 from morphlib.util import error_message_for_containerised_commandline
 import morphlib.gitversion
 
@@ -125,11 +124,7 @@ def ldconfig(runcmd, rootdir):  # pragma: no cover
 def download_depends(constituents, lac, rac, metadatas=None):
     for constituent in constituents:
         if not lac.has(constituent):
-            source = rac.get(constituent)
-            target = lac.put(constituent)
-            shutil.copyfileobj(source, target)
-            target.close()
-            source.close()
+            lac.copy_from_remote(constituent, rac)
         if metadatas is not None:
             for metadata in metadatas:
                 if not lac.has_artifact_metadata(constituent, metadata):
@@ -246,28 +241,6 @@ class ChunkBuilder(BuilderBase):
 
     '''Build chunk artifacts.'''
 
-    def create_devices(self, destdir): # pragma: no cover
-        '''Creates device nodes if the morphology specifies them'''
-        morphology = self.source.morphology
-        perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
-        if 'devices' in morphology and morphology['devices'] is not None:
-            for dev in morphology['devices']:
-                destfile = os.path.join(destdir, './' + dev['filename'])
-                mode = int(dev['permissions'], 8) & perms_mask
-                if dev['type'] == 'c':
-                    mode = mode | stat.S_IFCHR
-                elif dev['type'] == 'b':
-                    mode = mode | stat.S_IFBLK
-                else:
-                    raise IOError('Cannot create device node %s,'
-                                  'unrecognized device type "%s"'
-                                  % (destfile, dev['type']))
-                self.app.status(msg="Creating device node %s"
-                                    % destfile)
-                os.mknod(destfile, mode,
-                         os.makedev(dev['major'], dev['minor']))
-                os.chown(destfile, dev['uid'], dev['gid'])
-
     def build_and_cache(self):  # pragma: no cover
         with self.build_watch('overall-build'):
 
@@ -286,7 +259,6 @@ class ChunkBuilder(BuilderBase):
             try:
                 self.get_sources(builddir)
                 self.run_commands(builddir, destdir, temppath, stdout)
-                self.create_devices(destdir)
 
                 os.rename(temppath, logpath)
             except BaseException as e:
@@ -459,13 +431,23 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                with self.local_artifact_cache.put(chunk_artifact) as f:
-                    self.write_metadata(destdir, chunk_artifact_name,
-                                        parented_paths)
+                self.write_metadata(destdir, chunk_artifact_name,
+                                    parented_paths)
 
-                    self.app.status(msg='Creating chunk artifact %(name)s',
-                                    name=chunk_artifact_name)
-                    morphlib.bins.create_chunk(destdir, f, parented_paths)
+                self.app.status(msg='Creating chunk artifact %(name)s',
+                                name=chunk_artifact_name)
+                # TODO: This is not concurrency safe, bins.create_chunk will
+                #       fail if tempdir already exists (eg if another build
+                #       has created it).
+                tempdir = os.path.join(self.app.settings['tempdir'],
+                                       chunk_artifact.basename())
+                try:
+                    morphlib.bins.create_chunk(destdir, tempdir,
+                                               parented_paths)
+                    self.local_artifact_cache.put(tempdir, chunk_artifact)
+                finally:
+                    if os.path.isdir(tempdir):
+                        shutil.rmtree(tempdir)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -509,8 +491,13 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    with self.local_artifact_cache.put(a) as f:
+                    # TODO: This is not concurrency safe, put_stratum_artifact
+                    #       deletes temp which could be in use by another
+                    #       build.
+                    temp = os.path.join(self.app.settings['tempdir'], a.name)
+                    with open(temp, 'w+') as f:
                         json.dump([c.basename() for c in constituents], f)
+                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -532,64 +519,47 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
-                handle = self.local_artifact_cache.put(artifact)
-
                 try:
                     fs_root = self.staging_area.destdir(self.source)
-                    self.unpack_strata(fs_root)
-                    self.write_metadata(fs_root, a_name)
-                    self.run_system_integration_commands(fs_root)
-                    unslashy_root = fs_root[1:]
-                    def uproot_info(info):
-                        info.name = relpath(info.name, unslashy_root)
-                        if info.islnk():
-                            info.linkname = relpath(info.linkname,
-                                                    unslashy_root)
-                        return info
-                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
-                    self.app.status(msg='Constructing tarball of rootfs',
-                                    chatty=True)
-                    tar.add(fs_root, recursive=True, filter=uproot_info)
-                    tar.close()
+                    upperdir = self.staging_area.overlay_upperdir(
+                        self.source)
+                    editable_root = self.staging_area.overlaydir(self.source)
+                    workdir = os.path.join(self.staging_area.dirname,
+                                           'overlayfs-workdir')
+                    if not os.path.exists(workdir):
+                        os.makedirs(workdir)
+                    union_filesystem = self.app.settings['union-filesystem']
+                    morphlib.fsutils.overlay_mount(self.app.runcmd,
+                                                   'overlay-%s' % a_name,
+                                                   editable_root, fs_root,
+                                                   upperdir, workdir,
+                                                   union_filesystem)
+                    try:
+                        self.unpack_strata(fs_root)
+                        self.write_metadata(editable_root, a_name)
+                        self.run_system_integration_commands(editable_root)
+                        self.local_artifact_cache.put(editable_root, artifact)
+                    finally:
+                        morphlib.fsutils.unmount(self.app.runcmd,
+                                                 editable_root)
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
                                     error=True)
-                    handle.abort()
                     raise
-                else:
-                    handle.close()
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
 
-    def load_stratum(self, stratum_artifact):
-        '''Load a stratum from the local artifact cache.
-
-        Returns a list of ArtifactCacheReference instances for the chunks
-        contained in the stratum.
-
-        '''
-        cache = self.local_artifact_cache
-        with cache.get(stratum_artifact) as stratum_file:
-            try:
-                artifact_list = json.load(stratum_file,
-                                          encoding='unicode-escape')
-            except ValueError as e:
-                raise cliapp.AppException(
-                    'Corruption detected: %s while loading %s' %
-                    (e, cache.artifact_filename(stratum_artifact)))
-        return [ArtifactCacheReference(a) for a in artifact_list]
-
     def unpack_one_stratum(self, stratum_artifact, target):
         '''Unpack a single stratum into a target directory'''
 
         cache = self.local_artifact_cache
-        for chunk in self.load_stratum(stratum_artifact):
-            self.app.status(msg='Unpacking chunk %(basename)s',
+        chunks = morphlib.util.get_stratum_contents(cache, stratum_artifact)
+        for chunk in chunks:
+            self.app.status(msg='Checkout chunk %(basename)s',
                             basename=chunk.basename(), chatty=True)
-            with cache.get(chunk) as chunk_file:
-                morphlib.bins.unpack_binary_from_file(chunk_file, target)
+            cache.get(chunk, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -600,7 +570,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Unpacking strata to %(path)s',
+        self.app.status(msg='Checking out strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():
@@ -612,7 +582,8 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
                 # download the chunk artifacts if necessary
                 for stratum_artifact in self.source.dependencies:
-                    chunks = self.load_stratum(stratum_artifact)
+                    chunks = morphlib.util.get_stratum_contents(
+                        self.local_artifact_cache, stratum_artifact)
                     download_depends(chunks,
                                      self.local_artifact_cache,
                                      self.remote_artifact_cache)