deploy: Use OSTree to checkout the system for deployment

Now that we have an OSTree artifact cache, the deploy plugin needs to use
that to get the system to be deployed. Due to the changes in how we store
systems, we need to get the contents of each stratum then put the system
delta on top of that.

This is still much quicker than unpacking stuff from tarballs.

Change-Id: I209dc43a49bb00fa828907cb72715afc7061d43f

1 files changed, 101 insertions, 23 deletions

diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index b85c5e8f..9c2782e6 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -24,6 +24,15 @@ import uuid
 
 import cliapp
 import morphlib
+from morphlib.artifactcachereference import ArtifactCacheReference
+
+
+class NotYetBuiltError(morphlib.Error):
+
+    def __init__(self, name):
+        self.msg = ('Deployment failed as %s is not yet built.\n'
+                    'Please ensure the system is built before deployment.'
+                    % name)
 
 
 class DeployPlugin(cliapp.Plugin):
@@ -540,8 +549,98 @@ class DeployPlugin(cliapp.Plugin):
         except morphlib.extensions.ExtensionNotFoundError:
             pass
 
+    def checkout_stratum(self, path, artifact, lac, rac):
+        """Pull the chunks in a stratum, and checkout them into `path`.
+
+        This reads a stratum artifact and pulls the chunks it contains from
+        the remote into the local artifact cache if they are not already
+        cached locally. Each of these chunks is then checked out into `path`.
+
+        Also download the stratum metadata into the local cache, then place
+        it in the /baserock directory of the system checkout indicated by
+        `path`.
+
+        If any of the chunks have not been cached either locally or remotely,
+        a morphlib.remoteartifactcache.GetError is raised.
+
+        """
+        with open(lac.get(artifact), 'r') as stratum:
+            chunks = [ArtifactCacheReference(c) for c in json.load(stratum)]
+        morphlib.builder.download_depends(chunks, lac, rac)
+        for chunk in chunks:
+            self.app.status(msg='Checkout chunk %(name)s.',
+                            name=chunk.basename(), chatty=True)
+            lac.get(chunk, path)
+
+        metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name)
+        with lac.get_artifact_metadata(artifact, 'meta') as meta_src:
+            with morphlib.savefile.SaveFile(metadata, 'w') as meta_dst:
+                shutil.copyfileobj(meta_src, meta_dst)
+
+    def checkout_strata(self, path, artifact, lac, rac):
+        """Pull the dependencies of `artifact` and checkout them into `path`.
+
+        This assumes that `artifact` is a system artifact. If any of the
+        dependencies aren't cached remotely or locally, this raises a
+        morphlib.remoteartifactcache.GetError.
+
+        """
+        deps = artifact.source.dependencies
+        morphlib.builder.download_depends(deps, lac, rac)
+        for stratum in deps:
+            self.checkout_stratum(path, stratum, lac, rac)
+        morphlib.builder.ldconfig(self.app.runcmd, path)
+
+    def checkout_system(self, build_command, artifact, path):
+        """Checkout a system into `path`.
+
+        This checks out each of the strata into the directory given by `path`,
+        then checks out the system artifact into the same directory. This uses
+        OSTree's `union` checkout mode to overwrite duplicate files but not
+        need an empty directory. Artifacts which aren't cached locally are
+        fetched from the remote cache.
+
+        Raises a NotYetBuiltError if either the system artifact or any of the
+        chunk artifacts in the strata which make up the system aren't cached
+        either locally or remotely.
+
+        """
+        # Check if the system artifact is in the local or remote cache.
+        # If it isn't, we don't need to bother checking out strata before
+        # we fail.
+        if not (build_command.lac.has(artifact)
+                or build_command.rac.has(artifact)):
+            raise NotYetBuiltError(artifact.name)
+
+        # Checkout the strata involved in the artifact into a tempdir
+        self.app.status(msg='Checking out strata in system')
+        try:
+            self.checkout_strata(path, artifact,
+                                 build_command.lac, build_command.rac)
+
+            self.app.status(msg='Checking out system for configuration')
+            build_command.cache_artifacts_locally([artifact])
+            build_command.lac.get(artifact, path)
+        except (morphlib.ostreeartifactcache.NotCachedError,
+                morphlib.remoteartifactcache.GetError):
+            raise NotYetBuiltError(artifact.name)
+
+        self.app.status(
+            msg='System checked out at %(system_tree)s',
+            system_tree=path)
+
     def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref,
                      artifact, deployment_type, location, env):
+        """Checkout the artifact, create metadata and return the location.
+
+        This checks out the system into a temporary directory, and then mounts
+        this temporary directory alongside a different temporary directory
+        using a union filesystem. This allows changes to be made without
+        touching the checked out artifacts. The deployment metadata file is
+        created and then the directory at which the two temporary directories
+        are mounted is returned.
+
+        """
         # deployment_type, location and env are only used for saving metadata
 
         deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir)
@@ -561,25 +660,7 @@ class DeployPlugin(cliapp.Plugin):
         deploy_tree = os.path.join(deployment_dir,
                                    'overlay-deploy-%s' % artifact.name)
         try:
-            # Unpack the artifact (tarball) to a temporary directory.
-            self.app.status(msg='Unpacking system for configuration')
-
-            if build_command.lac.has(artifact):
-                f = build_command.lac.get(artifact)
-            elif build_command.rac.has(artifact):
-                build_command.cache_artifacts_locally([artifact])
-                f = build_command.lac.get(artifact)
-            else:
-                raise cliapp.AppException('Deployment failed as system is'
-                                          ' not yet built.\nPlease ensure'
-                                          ' the system is built before'
-                                          ' deployment.')
-            tf = tarfile.open(fileobj=f)
-            tf.extractall(path=system_tree)
-
-            self.app.status(
-                msg='System unpacked at %(system_tree)s',
-                system_tree=system_tree)
+            self.checkout_system(build_command, artifact, system_tree)
 
             union_filesystem = self.app.settings['union-filesystem']
             morphlib.fsutils.overlay_mount(self.app.runcmd,
@@ -602,10 +683,7 @@ class DeployPlugin(cliapp.Plugin):
         except Exception:
             if deploy_tree and os.path.exists(deploy_tree):
                 morphlib.fsutils.unmount(self.app.runcmd, deploy_tree)
-                shutil.rmtree(deploy_tree)
-            shutil.rmtree(system_tree)
-            shutil.rmtree(overlay_dir)
-            shutil.rmtree(work_dir)
+            shutil.rmtree(deployment_dir)
             raise
 
     def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir,