Allow the deployment of individual chunks/strata from systems

This commit allows the specification of one or more strata/chunks
in a deployment entry in a cluster morphology to deploy instead of
the full system if --partial is set. These are listed in a
'partial-deploy-components' field in each deployment definition.

The components must be in the system, and this only works for
tarball or sysroot deployments. It SHOULD NOT be used when
deploying production systems, as it has a number of limitations.

2 files changed, 125 insertions, 37 deletions

diff --git a/morphlib/app.py b/morphlib/app.py
index f7c07726..73587067 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -145,6 +145,10 @@ class Morph(cliapp.Application):
                               'always push temporary build branches to the '
                               'remote repository',
                               group=group_build)
+        self.settings.boolean(['partial'],
+                              'only build up to a given chunk',
+                              default=False,
+                              group=group_build)
         self.settings.choice (['local-changes'],
                               ['include', 'ignore'],
                               'the `build` and `deploy` commands detect '
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index c9890b13..d67b2441 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -13,6 +13,7 @@
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import collections
 import json
 import logging
 import os
@@ -385,7 +386,7 @@ class DeployPlugin(cliapp.Plugin):
                     name=name, email=email, build_uuid=build_uuid,
                     status=self.app.status)
             with pbb as (repo, commit, original_ref):
-                self.deploy_cluster(build_command, cluster_morphology,
+                self.deploy_cluster(sb, build_command, cluster_morphology,
                                     root_repo_dir, repo, commit, env_vars,
                                     deployments)
         else:
@@ -398,6 +399,11 @@ class DeployPlugin(cliapp.Plugin):
                                 deployments)
 
         self.app.status(msg='Finished deployment')
+        if self.app.settings['partial']:
+            self.app.status(msg='WARNING: This was a partial deployment. '
+                                'Configuration extensions have not been '
+                                'run. Applying the result to an existing '
+                                'system may not have reproducible results.')
 
     def validate_deployment_options(
             self, env_vars, all_deployments, all_subsystems):
@@ -415,21 +421,54 @@ class DeployPlugin(cliapp.Plugin):
                         'Variable referenced a non-existent deployment '
                         'name: %s' % var)
 
-    def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir,
-                       repo, commit, env_vars, deployments):
+    def deploy_cluster(self, sb, build_command, cluster_morphology,
+                       root_repo_dir, repo, commit, env_vars, deployments):
         # Create a tempdir for this deployment to work in
         deploy_tempdir = tempfile.mkdtemp(
             dir=os.path.join(self.app.settings['tempdir'], 'deployments'))
         try:
             for system in cluster_morphology['systems']:
-                self.deploy_system(build_command, deploy_tempdir,
+                self.deploy_system(sb, build_command, deploy_tempdir,
                                    root_repo_dir, repo, commit, system,
                                    env_vars, deployments,
                                    parent_location='')
         finally:
             shutil.rmtree(deploy_tempdir)
 
-    def deploy_system(self, build_command, deploy_tempdir,
+    def _sanitise_morphology_paths(self, paths, sb):
+        sanitised_paths = []
+        for path in paths:
+            path = morphlib.util.sanitise_morphology_path(path)
+            sanitised_paths.append(sb.relative_to_root_repo(path))
+        return sanitised_paths
+
+    def _find_artifacts(self, filenames, root_artifact):
+        found = collections.OrderedDict()
+        not_found = filenames
+        for a in root_artifact.walk():
+            if a.source.filename in filenames and a.source.name not in found:
+                found[a.source.name] = a
+                not_found.remove(a.source.filename)
+        return found, not_found
+
+    def _validate_partial_deployment(self, deployment_type,
+                                     artifact, component_names):
+        supported_types = ('tar', 'sysroot')
+        if deployment_type not in supported_types:
+            raise cliapp.AppException('Not deploying %s, --partial was '
+                                      'set and partial deployment only '
+                                      'supports %s deployments.' %
+                                      (artifact.source.name,
+                                       ', '.join(supported_types)))
+        components, not_found = self._find_artifacts(component_names,
+                                                     artifact)
+        if not_found:
+            raise cliapp.AppException('Components %s not found in system %s.' %
+                                      (', '.join(not_found),
+                                       artifact.source.name))
+        return components
+
+    def deploy_system(self, sb, build_command, deploy_tempdir,
                       root_repo_dir, build_repo, ref, system, env_vars,
                       deployment_filter, parent_location):
         sys_ids = set(system['deploy'].iterkeys())
@@ -475,6 +514,12 @@ class DeployPlugin(cliapp.Plugin):
                         raise morphlib.Error('"type" is undefined '
                                              'for system "%s"' % system_id)
 
+                    components = self._sanitise_morphology_paths(
+                        deploy_params.get('partial-deploy-components', []), sb)
+                    if self.app.settings['partial']:
+                        components = self._validate_partial_deployment(
+                            deployment_type, artifact, components)
+
                     location = final_env.pop('location', None)
                     if not location:
                         raise morphlib.Error('"location" is undefined '
@@ -488,9 +533,10 @@ class DeployPlugin(cliapp.Plugin):
                                                     root_repo_dir,
                                                     ref, artifact,
                                                     deployment_type,
-                                                    location, final_env)
+                                                    location, final_env,
+                                                    components=components)
                     for subsystem in system.get('subsystems', []):
-                        self.deploy_system(build_command, deploy_tempdir,
+                        self.deploy_system(sb, build_command, deploy_tempdir,
                                            root_repo_dir, build_repo,
                                            ref, subsystem, env_vars, [],
                                            parent_location=system_tree)
@@ -562,8 +608,48 @@ class DeployPlugin(cliapp.Plugin):
             self.checkout_stratum(path, stratum, lac, rac)
         morphlib.builder.ldconfig(self.app.runcmd, path)
 
+    def checkout_system(self, path, artifact, build_command):
+        # Checkout the strata involved in the artifact into a tempdir
+        self.app.status(msg='Checking out strata in system')
+        self.checkout_strata(path, artifact,
+                             build_command.lac, build_command.rac)
+
+        self.app.status(msg='Checking out system for configuration')
+        if build_command.lac.has(artifact):
+            build_command.lac.get(artifact, path)
+        elif build_command.rac.has(artifact):
+            build_command.cache_artifacts_locally([artifact])
+            build_command.lac.get(artifact, path)
+        else:
+            raise cliapp.AppException('Deployment failed as system is'
+                                      ' not yet built.\nPlease ensure'
+                                      ' the system is built before'
+                                      ' deployment.')
+
+    def checkout_components(self, path, components, bc):
+        if not components:
+            raise cliapp.AppException('Deployment failed as no components '
+                                      'were specified for deployment and '
+                                      '--partial was set.')
+        for name, artifact in components.iteritems():
+            deps = artifact.source.dependencies
+            morphlib.builder.download_depends(deps, bc.lac, bc.rac)
+            for dep in deps:
+                if dep.source.morphology['kind'] == 'stratum':
+                    self.checkout_stratum(path, dep, bc.lac, bc.rac)
+                elif dep.source.morphology['kind'] == 'chunk':
+                    self.app.status(msg='Checkout chunk %(name)s.',
+                                    name=dep.basename(), chatty=True)
+                    bc.lac.get(dep, path)
+            if artifact.source.morphology['kind'] == 'stratum':
+                self.checkout_stratum(path, artifact, bc.lac, bc.rac)
+            elif artifact.source.morphology['kind'] == 'chunk':
+                self.app.status(msg='Checkout chunk %(name)s.',
+                                name=name, chatty=True)
+                bc.lac.get(artifact, path)
+
     def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref,
-                     artifact, deployment_type, location, env):
+                     artifact, deployment_type, location, env, components=[]):
         # deployment_type, location and env are only used for saving metadata
 
         deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir)
@@ -583,26 +669,17 @@ class DeployPlugin(cliapp.Plugin):
         deploy_tree = os.path.join(deployment_dir,
                                    'overlay-deploy-%s' % artifact.name)
         try:
-            # Checkout the strata involved in the artifact into a tempdir
-            self.app.status(msg='Checking out strata in system')
-            self.checkout_strata(system_tree, artifact,
-                                 build_command.lac, build_command.rac)
-
-            self.app.status(msg='Checking out system for configuration')
-            if build_command.lac.has(artifact):
-                build_command.lac.get(artifact, system_tree)
-            elif build_command.rac.has(artifact):
-                build_command.cache_artifacts_locally([artifact])
-                build_command.lac.get(artifact, system_tree)
+            if self.app.settings['partial']:
+                self.checkout_components(system_tree, components,
+                                         build_command)
+                self.app.status(
+                    msg='Components %(components)s checkout out at %(path)s',
+                    components=', '.join(components), path=system_tree)
             else:
-                raise cliapp.AppException('Deployment failed as system is'
-                                          ' not yet built.\nPlease ensure'
-                                          ' the system is built before'
-                                          ' deployment.')
-
-            self.app.status(
-                msg='System checked out at %(system_tree)s',
-                system_tree=system_tree)
+                self.checkout_system(system_tree, artifact, build_command)
+                self.app.status(
+                    msg='System checked out at %(system_tree)s',
+                    system_tree=system_tree)
 
             union_filesystem = self.app.settings['union-filesystem']
             morphlib.fsutils.overlay_mount(self.app.runcmd,
@@ -640,15 +717,19 @@ class DeployPlugin(cliapp.Plugin):
 
         try:
             # Run configuration extensions.
-            self.app.status(msg='Configure system')
-            names = artifact.source.morphology['configuration-extensions']
-            for name in names:
-                self._run_extension(
-                    root_repo_dir,
-                    name,
-                    '.configure',
-                    [system_tree],
-                    env)
+            if not self.app.settings['partial']:
+                self.app.status(msg='Configure system')
+                names = artifact.source.morphology['configuration-extensions']
+                for name in names:
+                    self._run_extension(
+                        root_repo_dir,
+                        name,
+                        '.configure',
+                        [system_tree],
+                        env)
+            else:
+                self.app.status(msg='WARNING: Not running configuration '
+                                    'extensions as --partial is set!')
 
             # Run write extension.
             self.app.status(msg='Writing to device')
@@ -699,7 +780,7 @@ class DeployPlugin(cliapp.Plugin):
             raise cliapp.AppException(message)
 
     def create_metadata(self, system_artifact, root_repo_dir, deployment_type,
-                        location, env):
+                        location, env, components=[]):
         '''Deployment-specific metadata.
 
         The `build` and `deploy` operations must be from the same ref, so full
@@ -731,6 +812,9 @@ class DeployPlugin(cliapp.Plugin):
                 'commit': morphlib.gitversion.commit,
                 'version': morphlib.gitversion.version,
             },
+            'partial': self.app.settings['partial'],
         }
+        if self.app.settings['partial']:
+            meta['partial-components'] = components
 
         return meta