Use OSTree for hardlink and artifact cache and a CoW unionfs to make system artifacts fasterbaserock/richardmaw/ostree-squash

This replaces the artifact cache and the hardlink cache with an OSTree
repository, which is a great performance improvement when the cache
directory and temporary directory are on the same filesystem.

Additionally it can de-duplicate file contents.

When we construct system artifacts deploy them, the staging area needs
to be writable, so OSTree on its own is insufficient, as its hardlinks
require the root to be kept read-only.

To handle this we use either the in-kernel overlayfs or unionfs-fuse,
though there is no automatic fall-back and it needs to be specified
manually.

To support distributed building, the artifact cache is extended to
support an OSTree repository.

Unfortunately cross-bootstrap is not expected to work with these changes
at this point in time.

IMPORTANT NOTE: We are well aware that this patch is too large to be
comprehensible. We intend to revert and apply a cleaned up series when
it is ready.

Change-Id: I693bb752500dab3c6db3b97393689239ae7071a8

1 files changed, 97 insertions, 23 deletions

diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index ea84d9ec..231fa868 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -25,6 +25,15 @@ import warnings
 
 import cliapp
 import morphlib
+from morphlib.artifactcachereference import ArtifactCacheReference
+
+
+class NotYetBuiltError(morphlib.Error):
+
+    def __init__(self, name):
+        self.msg = ('Deployment failed as %s is not yet built.\n'
+                    'Please ensure the system is built before deployment.'
+                    % name)
 
 
 def configuration_for_system(system_id, vars_from_commandline,
@@ -419,6 +428,8 @@ class DeployPlugin(cliapp.Plugin):
         system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph'])
         self.app.status_prefix = system_status_prefix
         try:
+            system_tree = None
+
             # Find the artifact to build
             morph = morphlib.util.sanitise_morphology_path(system['morph'])
             srcpool = build_command.create_source_pool(build_repo, ref, morph)
@@ -467,6 +478,9 @@ class DeployPlugin(cliapp.Plugin):
                                              system_tree, deploy_location)
                 finally:
                     self.app.status_prefix = system_status_prefix
+                    if system_tree and os.path.exists(system_tree):
+                        morphlib.fsutils.unmount(self.app.runcmd, system_tree)
+                        shutil.rmtree(system_tree)
         finally:
             self.app.status_prefix = old_status_prefix
 
@@ -525,46 +539,106 @@ class DeployPlugin(cliapp.Plugin):
         except morphlib.extensions.ExtensionNotFoundError:
             pass
 
+    def checkout_system(self, build_command, artifact, path):
+        """Checkout a system into `path`.
+
+        This checks out the system artifact into the directory given by
+        `path`. If the system is not in the local cache, it is first fetched
+        from the remote cache.
+
+        Raises a NotYetBuiltError if the system artifact isn't cached either
+        locally or remotely.
+
+        """
+        try:
+            self.app.status(msg='Checking out system for configuration')
+            build_command.cache_artifacts_locally([artifact])
+            build_command.lac.get(artifact, path)
+            self.create_device_nodes(artifact, path)
+        except (morphlib.ostreeartifactcache.NotCachedError,
+                morphlib.remoteartifactcache.GetError):
+            raise NotYetBuiltError(artifact.name)
+
+        self.app.status(
+            msg='System checked out at %(system_tree)s',
+            system_tree=path)
+
+    def create_device_nodes(self, artifact, path):
+        self.fix_chunk_build_mode(artifact)
+        for a in artifact.walk():
+            morph = a.source.morphology
+            if morph['kind'] == 'chunk' and \
+                morph['build-mode'] != 'bootstrap':
+                morphlib.util.create_devices(a.source.morphology, path)
+
+    def fix_chunk_build_mode(self, system_artifact):
+        """Give each chunk's in-memory morpholgy the correct build-mode."""
+        strata = set(a for a in system_artifact.walk()
+                     if a.source.morphology['kind'] == 'stratum')
+        chunks = set(a for a in system_artifact.walk()
+                     if a.source.morphology['kind'] == 'chunk')
+        for chunk in chunks:
+            for stratum in strata:
+                for spec in stratum.source.morphology['chunks']:
+                    if chunk.source.morphology['name'] == spec['name']:
+                        chunk.source.morphology['build-mode'] = \
+                            spec['build-mode']
+
     def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref,
                      artifact, deployment_type, location, env):
+        """Checkout the artifact, create metadata and return the location.
+
+        This checks out the system into a temporary directory, and then mounts
+        this temporary directory alongside a different temporary directory
+        using a union filesystem. This allows changes to be made without
+        touching the checked out artifacts. The deployment metadata file is
+        created and then the directory at which the two temporary directories
+        are mounted is returned.
+
+        """
         # deployment_type, location and env are only used for saving metadata
 
-        # Create a tempdir to extract the rootfs in
-        system_tree = tempfile.mkdtemp(dir=deploy_tempdir)
+        deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir)
 
+        # Create a tempdir to extract the rootfs in
+        system_tree = tempfile.mkdtemp(dir=deployment_dir)
+
+        # Create temporary directory for overlayfs
+        overlay_dir = os.path.join(deployment_dir,
+                                   '%s-upperdir' % artifact.name)
+        if not os.path.exists(overlay_dir):
+            os.makedirs(overlay_dir)
+        work_dir = os.path.join(deployment_dir, '%s-workdir' % artifact.name)
+        if not os.path.exists(work_dir):
+            os.makedirs(work_dir)
+
+        deploy_tree = os.path.join(deployment_dir,
+                                   'overlay-deploy-%s' % artifact.name)
         try:
-            # Unpack the artifact (tarball) to a temporary directory.
-            self.app.status(msg='Unpacking system for configuration')
-
-            if build_command.lac.has(artifact):
-                f = build_command.lac.get(artifact)
-            elif build_command.rac.has(artifact):
-                build_command.cache_artifacts_locally([artifact])
-                f = build_command.lac.get(artifact)
-            else:
-                raise cliapp.AppException('Deployment failed as system is'
-                                          ' not yet built.\nPlease ensure'
-                                          ' the system is built before'
-                                          ' deployment.')
-            tf = tarfile.open(fileobj=f)
-            tf.extractall(path=system_tree)
+            self.checkout_system(build_command, artifact, system_tree)
 
-            self.app.status(
-                msg='System unpacked at %(system_tree)s',
-                system_tree=system_tree)
+            union_filesystem = self.app.settings['union-filesystem']
+            morphlib.fsutils.overlay_mount(self.app.runcmd,
+                                           'overlay-deploy-%s' %
+                                           artifact.name,
+                                           deploy_tree, system_tree,
+                                           overlay_dir, work_dir,
+                                           union_filesystem)
 
             self.app.status(
                 msg='Writing deployment metadata file')
             metadata = self.create_metadata(
                     artifact, root_repo_dir, deployment_type, location, env)
             metadata_path = os.path.join(
-                    system_tree, 'baserock', 'deployment.meta')
+                    deploy_tree, 'baserock', 'deployment.meta')
             with morphlib.savefile.SaveFile(metadata_path, 'w') as f:
                 json.dump(metadata, f, indent=4,
                           sort_keys=True, encoding='unicode-escape')
-            return system_tree
+            return deploy_tree
         except Exception:
-            shutil.rmtree(system_tree)
+            if deploy_tree and os.path.exists(deploy_tree):
+                morphlib.fsutils.unmount(self.app.runcmd, deploy_tree)
+            shutil.rmtree(deployment_dir)
             raise
 
     def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir,