Implement partial builds

This commit allows you to build only a single chunk or stratum and its
dependencies, instead of a whole system. This will be useful for
developers who want to quickly check the build commands of a chunk for
example. You can give more than one chunk or stratum to the command,
and the build will run up to the latest one, building the dependencies
of both.

An example of the usage:

    morph build --partial systems/devel-system-x86_64-generic.morph \
        strata/build-essential.morph strata/core/curl.morph

This will only build up to curl in the core stratum.

Change-Id: I945a5ed8187f194fc41bcac8fa1f51cc5f9a5392

3 files changed, 108 insertions, 17 deletions

diff --git a/morphlib/app.py b/morphlib/app.py
index c8fe397d..9d847b64 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -138,6 +138,10 @@ class Morph(cliapp.Application):
                               'always push temporary build branches to the '
                               'remote repository',
                               group=group_build)
+        self.settings.boolean(['partial'],
+                              'only build up to a given chunk',
+                              default=False,
+                              group=group_build)
         self.settings.choice (['local-changes'],
                               ['include', 'ignore'],
                               'the `build` and `deploy` commands detect '
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index be8a1507..40510560 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -271,7 +271,8 @@ class BuildCommand(object):
     def build_in_order(self, root_artifact):
         '''Build everything specified in a build order.'''
 
-        self.app.status(msg='Building a set of sources')
+        self.app.status(msg='Starting build of %(name)s',
+                        name=root_artifact.source.name)
         build_env = root_artifact.build_env
         ordered_sources = list(self.get_ordered_sources(root_artifact.walk()))
         old_prefix = self.app.status_prefix
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 2cc395fc..06f6ced7 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -13,21 +13,31 @@
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-import cliapp
+import collections
 import contextlib
 import uuid
 import logging
 
+import cliapp
+
 import morphlib
 
 
+class ComponentNotInSystemError(morphlib.Error):
+
+    def __init__(self, components, system):
+        components = ', '.join(components)
+        self.msg = 'Components %s are not in %s' % (components, system)
+
+
 class BuildPlugin(cliapp.Plugin):
 
     def enable(self):
         self.app.add_subcommand('build-morphology', self.build_morphology,
-                                arg_synopsis='(REPO REF FILENAME)...')
+                                arg_synopsis='REPO REF FILENAME '
+                                '[COMPONENT...]')
         self.app.add_subcommand('build', self.build,
-                                arg_synopsis='SYSTEM')
+                                arg_synopsis='SYSTEM [COMPONENT...]')
         self.app.add_subcommand('distbuild-morphology',
                                 self.distbuild_morphology,
                                 arg_synopsis='SYSTEM')
@@ -92,6 +102,8 @@ class BuildPlugin(cliapp.Plugin):
         * `REPO` is a git repository URL.
         * `REF` is a branch or other commit reference in that repository.
         * `FILENAME` is a morphology filename at that ref.
+        * `COMPONENT...` is one or more chunks or strata to build. This
+          is only used if `--partial` is set.
 
         You probably want `morph build` instead. However, in some
         cases it is more convenient to not have to create a Morph
@@ -104,8 +116,14 @@ class BuildPlugin(cliapp.Plugin):
 
         Example:
 
-            morph build-morphology baserock:baserock/definitions \
-                master devel-system-x86_64-generic.morph
+            morph build-morphology baserock:baserock/definitions \\
+                master systems/devel-system-x86_64-generic.morph
+
+        Partial build example:
+
+            morph build-morphology --partial baserock:baserock/definitions \\
+                master systems/devel-system-x86_64-generic.morph \\
+                strata/build-essential.morph
 
         '''
 
@@ -117,8 +135,12 @@ class BuildPlugin(cliapp.Plugin):
             self.app.settings['cachedir-min-space'])
 
         build_command = morphlib.buildcommand.BuildCommand(self.app)
-        for repo_name, ref, filename in self.app.itertriplets(args):
-            build_command.build(repo_name, ref, filename)
+        repo, ref, filename = args[0:3]
+        filename = morphlib.util.sanitise_morphology_path(filename)
+        component_filenames = [morphlib.util.sanitise_morphology_path(name)
+                               for name in args[3:]]
+        self.start_build(repo, ref, build_command, filename,
+                         component_filenames)
 
     def build(self, args):
         '''Build a system image in the current system branch
@@ -126,6 +148,8 @@ class BuildPlugin(cliapp.Plugin):
         Command line arguments:
 
         * `SYSTEM` is the name of the system to build.
+        * `COMPONENT...` is one or more chunks or strata to build. This
+          is only used if `--partial` is set.
 
         This builds a system image, and any of its components that
         need building.  The system name is the basename of the system
@@ -145,11 +169,16 @@ class BuildPlugin(cliapp.Plugin):
 
         Example:
 
-            morph build devel-system-x86_64-generic.morph
+            morph build systems/devel-system-x86_64-generic.morph
+
+        Partial build example:
+
+            morph build --partial systems/devel-system-x86_64-generic.morph \\
+                strata/build-essential.morph
 
         '''
 
-        if len(args) != 1:
+        if len(args) != 1 and not self.app.settings['partial']:
             raise cliapp.AppException('morph build expects exactly one '
                                       'parameter: the system to build')
 
@@ -165,6 +194,10 @@ class BuildPlugin(cliapp.Plugin):
 
         system_filename = morphlib.util.sanitise_morphology_path(args[0])
         system_filename = sb.relative_to_root_repo(system_filename)
+        if self.app.settings['partial']:
+            component_filenames = self._sanitise_morphology_paths(args[1:], sb)
+        else:
+            component_filenames = []
 
         logging.debug('System branch is %s' % sb.root_directory)
 
@@ -178,11 +211,14 @@ class BuildPlugin(cliapp.Plugin):
             build_command = morphlib.buildcommand.BuildCommand(self.app)
 
         if self.app.settings['local-changes'] == 'include':
-            self._build_with_local_changes(build_command, sb, system_filename)
+            self._build_with_local_changes(build_command, sb, system_filename,
+                                           component_filenames)
         else:
-            self._build_local_commit(build_command, sb, system_filename)
+            self._build_local_commit(build_command, sb, system_filename,
+                                     component_filenames)
 
-    def _build_with_local_changes(self, build_command, sb, system_filename):
+    def _build_with_local_changes(self, build_command, sb, system_filename,
+                                  component_filenames):
         '''Construct a branch including user's local changes, and build that.
 
         It is often a slow process to check all repos in the system branch for
@@ -211,10 +247,11 @@ class BuildPlugin(cliapp.Plugin):
                 name=name, email=email, build_uuid=build_uuid,
                 status=self.app.status)
         with pbb as (repo, commit, original_ref):
-            build_command.build(repo, commit, system_filename,
-                                original_ref=original_ref)
+            self.start_build(repo, commit, build_command, system_filename,
+                             component_filenames, original_ref=original_ref)
 
-    def _build_local_commit(self, build_command, sb, system_filename):
+    def _build_local_commit(self, build_command, sb, system_filename,
+                            component_filenames):
         '''Build whatever commit the user has checked-out locally.
 
         This ignores any uncommitted changes. Also, if the user has a commit
@@ -242,4 +279,53 @@ class BuildPlugin(cliapp.Plugin):
         definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path)
         commit = definitions_repo.resolve_ref_to_commit(ref)
 
-        build_command.build(root_repo_url, commit, system_filename)
+        self.start_build(root_repo_url, commit, build_command,
+                         system_filename, component_filenames)
+
+    def _sanitise_morphology_paths(self, paths, sb):
+        sanitised_paths = []
+        for path in paths:
+            path = morphlib.util.sanitise_morphology_path(path)
+            sanitised_paths.append(sb.relative_to_root_repo(path))
+        return sanitised_paths
+
+    def _find_artifacts(self, filenames, root_artifact):
+        found = collections.OrderedDict()
+        not_found = filenames
+        for a in root_artifact.walk():
+            if a.source.filename in filenames and a.source.name not in found:
+                found[a.source.name] = a
+                not_found.remove(a.source.filename)
+        return found, not_found
+
+    def start_build(self, repo, commit, bc, system_filename,
+                    component_filenames, original_ref=None):
+        '''Actually run the build.
+
+        If --partial was set on the command line, we expect that there is a
+        list of chunks for us to build. Decide on the correct order to build
+        these in and build each one in turn.
+
+        '''
+        if not self.app.settings['partial']:
+            if original_ref:
+                bc.build(repo, commit, system_filename,
+                         original_ref=original_ref)
+            else:
+                bc.build(repo, commit, system_filename)
+            return
+
+        self.app.status(msg='Deciding on task order')
+        srcpool = bc.create_source_pool(repo, commit, system_filename)
+        root = bc.resolve_artifacts(srcpool)
+        components, not_found = self._find_artifacts(component_filenames, root)
+        if not_found:
+            raise ComponentNotInSystemError(not_found, system_filename)
+
+        for name, component in components.iteritems():
+            component.build_env = root.build_env
+            bc.build_in_order(component)
+            self.app.status(msg='%(kind)s %(name)s is cached at %(path)s',
+                            kind=component.source.morphology['kind'],
+                            name=name,
+                            path=bc.lac.artifact_filename(component))