Use DefinitionsRepo in build command

This allows using `morph build`, `morph distbuild` and `morph
distbuild-start` from any Git checkout of a definitions.git repo, so
nobody needs to use `morph checkout` or `morph branch` if they don't
want to.

Change-Id: I5fdfae0f8bec1953893e26f0d227e289da11fa84

3 files changed, 79 insertions, 152 deletions

diff --git a/morphlib/definitions_repo.py b/morphlib/definitions_repo.py
index e0c7fb85..511bc523 100644
--- a/morphlib/definitions_repo.py
+++ b/morphlib/definitions_repo.py
@@ -202,15 +202,15 @@ class DefinitionsRepo(gitdir.GitDirectory):
                 build_ref_prefix=build_ref_prefix, git_user_name=git_user_name,
                 git_user_email=git_user_email, status_cb=status_cb)
             with temporary_branch as (repo_url, commit, original_ref):
+                if status_cb:
+                    status_cb(msg='Deciding on task order')
+
                 yield morphlib.sourceresolver.create_source_pool(
                     lrc, rrc, repo_url, commit, system_filename,
                     cachedir=cachedir, original_ref=original_ref,
                     update_repos=update_repos, status_cb=status_cb)
         else:
-            if self.system_branch:
-                repo_url = self.systembranch.root_repository
-            else:
-                repo_url = self.get_remote('origin')
+            repo_url = self.remote_url
             commit = self.resolve_ref_to_commit(ref)
 
             if status_cb:
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 2a60d9bb..70914443 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -615,6 +615,9 @@ class GitDirectory(object):
                 # ref wasn't a branch, can't have upstream
                 # treat it the same as no upstream for convenience
                 return None
+            elif 'No such branch' in emsg:
+                # Same as above
+                return None
             elif 'No upstream configured for branch' in emsg:
                 return None
             raise
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index a7eed59a..5619dd53 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -14,9 +14,7 @@
 
 
 import collections
-import contextlib
 import uuid
-import logging
 
 import cliapp
 
@@ -72,26 +70,17 @@ class BuildPlugin(cliapp.Plugin):
         See 'help distbuild' and 'help build-morphology' for more information.
 
         '''
-
         MINARGS = 3
 
         if len(args) < MINARGS:
             raise cliapp.AppException(self._cmd_usage('distbuild-morphology'))
 
         repo, ref, filename = args[0:MINARGS]
+        filename = morphlib.util.sanitise_morphology_path(filename)
 
-        addr = self.app.settings['controller-initiator-address']
-        port = self.app.settings['controller-initiator-port']
-
-        self.use_distbuild = True
-        build_command = morphlib.buildcommand.InitiatorBuildCommand(
-            self.app, addr, port)
+        component_names = args[MINARGS:]
 
-        filename = morphlib.util.sanitise_morphology_path(filename)
-        component_names = [morphlib.util.sanitise_morphology_path(name)
-                               for name in args[MINARGS:]]
-        self.start_build(repo, ref, build_command, filename,
-                         component_names)
+        self._distbuild(repo, ref, filename, component_names=component_names)
 
     def distbuild(self, args):
         '''Distbuild a system image in the current system branch
@@ -122,14 +111,40 @@ class BuildPlugin(cliapp.Plugin):
             morph distbuild devel-system-x86_64-generic.morph
 
         '''
-
         MINARGS = 1
 
         if len(args) < MINARGS:
             raise cliapp.AppException(self._cmd_usage('distbuild'))
 
-        self.use_distbuild = True
-        self.build(args)
+        definitions_repo = morphlib.definitions_repo.open(
+            '.', search_for_root=True, search_workspace=True, app=self.app)
+
+        filename = args[0]
+        filename = morphlib.util.sanitise_morphology_path(filename)
+        filename = definitions_repo.relative_path(filename, cwd='.')
+
+        component_names = args[MINARGS:]
+
+        if self.app.settings['local-changes'] == 'include':
+            # Create a temporary branch with any local changes, and push it to
+            # the shared Git server. This is a convenience for developers, who
+            # otherwise need to commit and push each change manually in order
+            # for distbuild to see it. It renders the build unreproducible, as
+            # the branch is deleted after being built, so this feature should
+            # only be used during development!
+            build_uuid = uuid.uuid4().hex
+            branch = definitions_repo.branch_with_local_changes(
+                build_uuid, push=True)
+            with branch as (repo_url, commit, original_ref):
+                self._distbuild(repo_url, commit, filename,
+                                original_ref=original_ref,
+                                component_names=component_names)
+        else:
+            ref = definitions_repo.HEAD
+            commit = definitions_repo.resolve_ref_to_commit(ref)
+            self._distbuild(definitions_repo.remote_url, commit, filename,
+                            original_ref=original_ref,
+                            component_names=component_names)
 
     def distbuild_start(self, args):
         '''Distbuild a system image without a lasting client-server connection.
@@ -145,15 +160,21 @@ class BuildPlugin(cliapp.Plugin):
         See `morph help distbuild` for more information and example usage.
 
         '''
+        self.allow_detach = True
+        self.distbuild(self, args)
 
-        MINARGS = 1
+    def _distbuild(self, repo_url, commit, filename, original_ref=None,
+                   component_names=[]):
+        '''Request a distributed build of a given system definition.'''
 
-        if len(args) < MINARGS:
-            raise cliapp.AppException(self._cmd_usage('distbuild-start'))
+        addr = self.app.settings['controller-initiator-address']
+        port = self.app.settings['controller-initiator-port']
 
-        self.use_distbuild = True
-        self.allow_detach = True
-        self.build(args)
+        build_command = morphlib.buildcommand.InitiatorBuildCommand(
+            self.app, addr, port, allow_detach=self.allow_detach)
+        build_command.build(
+            repo_url, commit, filename, original_ref=original_ref,
+            component_names=component_names)
 
     def build_morphology(self, args):
         '''Build a system, outside of a system branch.
@@ -187,13 +208,14 @@ class BuildPlugin(cliapp.Plugin):
                 build-essential
 
         '''
-
         MINARGS = 3
 
         if len(args) < MINARGS:
             raise cliapp.AppException(self._cmd_usage('build-morphology'))
 
         repo, ref, filename = args[0:MINARGS]
+        filename = morphlib.util.sanitise_morphology_path(filename)
+        component_names = args[MINARGS:]
 
         # Raise an exception if there is not enough space
         morphlib.util.check_disk_available(
@@ -203,15 +225,11 @@ class BuildPlugin(cliapp.Plugin):
             self.app.settings['cachedir-min-space'])
 
         build_command = morphlib.buildcommand.BuildCommand(self.app)
-
-        filename = morphlib.util.sanitise_morphology_path(filename)
-        component_names = [morphlib.util.sanitise_morphology_path(name)
-                               for name in args[MINARGS:]]
-        self.start_build(repo, ref, build_command, filename,
-                         component_names)
+        srcpool = build_command.create_source_pool(repo, ref, filename)
+        self._build(srcpool, filename, component_names=component_names)
 
     def build(self, args):
-        '''Build a system image in the current system branch
+        '''Build a system image in the current definitions repo.
 
         Command line arguments:
 
@@ -245,112 +263,32 @@ class BuildPlugin(cliapp.Plugin):
                 build-essential
 
         '''
-
         MINARGS = 1
 
         if len(args) < MINARGS:
             raise cliapp.AppException(self._cmd_usage('build'))
 
-        if not self.use_distbuild:
-            # Raise an exception if there is not enough space
-            morphlib.util.check_disk_available(
-                self.app.settings['tempdir'],
-                self.app.settings['tempdir-min-space'],
-                self.app.settings['cachedir'],
-                self.app.settings['cachedir-min-space'])
+        definitions_repo = morphlib.definitions_repo.open(
+            '.', search_for_root=True, search_workspace=True, app=self.app)
 
-        ws = morphlib.workspace.open('.')
-        sb = morphlib.sysbranchdir.open_from_within('.')
-
-        system_filename = morphlib.util.sanitise_morphology_path(args[0])
-        system_filename = sb.relative_to_root_repo(system_filename)
+        filename = args[0]
+        filename = morphlib.util.sanitise_morphology_path(filename)
+        filename = definitions_repo.relative_path(filename, cwd='.')
         component_names = args[MINARGS:]
 
-        logging.debug('System branch is %s' % sb.root_directory)
-
-        if self.use_distbuild:
-            addr = self.app.settings['controller-initiator-address']
-            port = self.app.settings['controller-initiator-port']
-
-            build_command = morphlib.buildcommand.InitiatorBuildCommand(
-                self.app, addr, port, self.allow_detach)
-        else:
-            build_command = morphlib.buildcommand.BuildCommand(self.app)
-
-        if self.app.settings['local-changes'] == 'include':
-            self._build_with_local_changes(build_command, sb, system_filename,
-                                           component_names)
-        else:
-            self._build_local_commit(build_command, sb, system_filename,
-                                     component_names)
-
-    def _build_with_local_changes(self, build_command, sb, system_filename,
-                                  component_names):
-        '''Construct a branch including user's local changes, and build that.
-
-        It is often a slow process to check all repos in the system branch for
-        local changes. However, when using a distributed build cluster, all
-        code being built must be pushed to the associated Trove, and it can be
-        helpful to have this automated as part of the `morph build` command.
-
-        '''
-        build_uuid = uuid.uuid4().hex
-
-        loader = morphlib.morphloader.MorphologyLoader()
-        push = self.app.settings['push-build-branches']
-        name = morphlib.git.get_user_name(self.app.runcmd)
-        email = morphlib.git.get_user_email(self.app.runcmd)
-        build_ref_prefix = self.app.settings['build-ref-prefix']
-
-        self.app.status(msg='Looking for uncommitted changes (pass '
-                            '--local-changes=ignore to skip)')
-
-        self.app.status(msg='Collecting morphologies involved in '
-                            'building %(system)s from %(branch)s',
-                            chatty=True,
-                            system=system_filename,
-                            branch=sb.system_branch_name)
-
-        bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix)
-        pbb = morphlib.buildbranch.pushed_build_branch(
-                bb, loader=loader, changes_need_pushing=push,
-                name=name, email=email, build_uuid=build_uuid,
-                status=self.app.status)
-        with pbb as (repo, commit, original_ref):
-            self.start_build(repo, commit, build_command, system_filename,
-                             component_names, original_ref=original_ref)
-
-    def _build_local_commit(self, build_command, sb, system_filename,
-                            component_names):
-        '''Build whatever commit the user has checked-out locally.
-
-        This ignores any uncommitted changes. Also, if the user has a commit
-        checked out locally that hasn't been pushed to the Trove that Morph is
-        configured to work with, the build will fail in this sort of way:
-
-            ERROR: Ref c55b853d92a52a5b5fe62edbfbf351169eb79c0a is an invalid
-            reference for repo
-            git://git.baserock.org/baserock/baserock/definitions
-
-        The build process doesn't use the checked-out definitions repo at all,
-        except to resolve the checked-out commit (HEAD). Instead, it uses the
-        cached version of the definitions repo, updating the cache if
-        necessary.
-
-        We don't detect and warn the user about any uncommitted changes because
-        doing so is slow when there are no changes (around 5 seconds on my
-        machine for Baserock's definitions.git).
-
-        '''
-        root_repo_url = sb.get_config('branch.root')
-        ref = sb.get_config('branch.name')
-
-        definitions_repo_path = sb.get_git_directory_name(root_repo_url)
-        definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path)
-        commit = definitions_repo.resolve_ref_to_commit(ref)
+        # Raise an exception if there is not enough space
+        morphlib.util.check_disk_available(
+            self.app.settings['tempdir'],
+            self.app.settings['tempdir-min-space'],
+            self.app.settings['cachedir'],
+            self.app.settings['cachedir-min-space'])
 
-        self.start_build(root_repo_url, commit, build_command,
-                         system_filename, component_names, original_ref=ref)
+        local_changes = self.app.settings['local-changes']
+        source_pool_context = definitions_repo.source_pool(
+            definitions_repo.HEAD, filename,
+            include_local_changes=(local_changes == 'include'))
+        with source_pool_context as source_pool:
+            self._build(source_pool, filename, component_names=component_names)
 
     def _find_artifacts(self, names, root_artifact):
         found = collections.OrderedDict()
@@ -362,31 +300,17 @@ class BuildPlugin(cliapp.Plugin):
                 not_found.remove(name)
         return found, not_found
 
-    def start_build(self, repo, commit, bc, system_filename,
-                    component_names, original_ref=None):
-        '''Actually run the build.
+    def _build(self, source_pool, filename, component_names=None):
+        '''Perform a local build of a given system definition.'''
 
-        If a set of components was given, only build those. Otherwise,
-        build the whole system.
-
-        '''
-        if self.use_distbuild:
-            bc.build(repo, commit, system_filename,
-                     original_ref=original_ref,
-                     component_names=component_names)
-            return
-
-        self.app.status(msg='Deciding on task order')
-        srcpool = bc.create_source_pool(repo, commit, system_filename,
-                                        original_ref)
-        bc.validate_sources(srcpool)
-        root = bc.resolve_artifacts(srcpool)
+        bc = morphlib.buildcommand.BuildCommand(self.app)
+        bc.validate_sources(source_pool)
+        root = bc.resolve_artifacts(source_pool)
         if not component_names:
             component_names = [root.source.name]
         components, not_found = self._find_artifacts(component_names, root)
         if not_found:
-            raise ComponentNotInSystemError(not_found, system_filename)
-
+            raise ComponentNotInSystemError(not_found, filename)
         for name, component in components.iteritems():
             component.build_env = root.build_env
             bc.build_in_order(component)