diff options
-rw-r--r-- | morphlib/buildbranch.py | 134 | ||||
-rw-r--r-- | morphlib/gitdir.py | 14 | ||||
-rw-r--r-- | morphlib/plugins/build_plugin.py | 39 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 41 |
4 files changed, 129 insertions, 99 deletions
diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 7aa75dc1..885f5cf8 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -15,6 +15,7 @@ import collections +import contextlib import os import urlparse @@ -48,10 +49,9 @@ class BuildBranch(object): # would be better to not use local repositories and temporary refs, # so building from a workspace appears to be identical to using # `morph build-morphology` - def __init__(self, sb, build_ref_prefix, push_temporary): + def __init__(self, sb, build_ref_prefix): self._sb = sb - self._push_temporary = push_temporary self._cleanup = collections.deque() self._to_push = {} @@ -86,13 +86,16 @@ class BuildBranch(object): def add_uncommitted_changes(self, add_cb=lambda **kwargs: None): '''Add any uncommitted changes to temporary build GitIndexes''' + changes_made = False for gd, (build_ref, index) in self._to_push.iteritems(): changed = [to_path for code, to_path, from_path in index.get_uncommitted_changes()] if not changed: continue add_cb(gd=gd, build_ref=gd, changed=changed) + changes_made = True index.add_files_from_working_tree(changed) + return changes_made @staticmethod def _hash_morphologies(gd, morphologies, loader): @@ -102,7 +105,8 @@ class BuildBranch(object): sha1 = gd.store_blob(loader.save_to_string(morphology)) yield 0100644, sha1, morphology.filename - def inject_build_refs(self, loader, inject_cb=lambda **kwargs: None): + def inject_build_refs(self, loader, use_local_repos, + inject_cb=lambda **kwargs: None): '''Update system and stratum morphologies to point to our branch. For all edited repositories, this alter the temporary GitIndex @@ -133,7 +137,7 @@ class BuildBranch(object): spec['repo'] = None spec['ref'] = None return True - if not self._push_temporary: + if use_local_repos: spec['repo'] = urlparse.urljoin('file://', gd.dirname) spec['ref'] = build_ref return True @@ -143,6 +147,8 @@ class BuildBranch(object): if any(m.dirty for m in morphs.morphologies): inject_cb(gd=self._root) + # TODO: Prevent it hashing unchanged morphologies, while still + # hashing uncommitted ones. self._root_index.add_files_from_index_info( self._hash_morphologies(self._root, morphs.morphologies, loader)) @@ -177,12 +183,18 @@ class BuildBranch(object): with morphlib.branchmanager.LocalRefManager() as lrm: for gd, (build_ref, index) in self._to_push.iteritems(): - commit_cb(gd=gd, build_ref=build_ref) tree = index.write_tree() try: parent = gd.resolve_ref_to_commit(build_ref) except morphlib.gitdir.InvalidRefError: parent = gd.resolve_ref_to_commit(gd.HEAD) + else: + # Skip updating ref if we already have a temporary + # build branch and have this tree on the branch + if tree == gd.resolve_ref_to_tree(build_ref): + continue + + commit_cb(gd=gd, build_ref=build_ref) commit = gd.commit_tree(tree, parent=parent, committer_name=committer_name, @@ -202,47 +214,55 @@ class BuildBranch(object): # a problem. lrm.update(gd, build_ref, commit, old_commit) - def push_build_branches(self, push_cb=lambda **kwargs: None): - '''Push all temporary build branches to the remote repositories. + def get_unpushed_branches(self): + '''Work out which, if any, local branches need to be pushed to build + + NOTE: This assumes that the refs in the morphologies and the + refs in the local checkouts match. - This is a no-op if the BuildBranch was constructed with - `push_temporary` as False, so that the code flow for the user of - the BuildBranch can be the same when it can be pushed as when - it can't. + ''' + for gd, (build_ref, index) in self._to_push.iteritems(): + remote = gd.get_remote('origin') + head_ref = gd.disambiguate_ref(gd.HEAD) + head_sha1 = gd.resolve_ref_to_commit(head_ref) + pushed_refs = sorted( + (remote_ref + for remote_sha1, remote_ref in remote.ls() + # substring match of refs, since ref may be a tag, + # in which case it would end with ^{} + if remote_sha1 == head_sha1 and head_ref in remote_ref), + key=len) + if not pushed_refs: + yield gd + def push_build_branches(self, push_cb=lambda **kwargs: None): + '''Push all temporary build branches to the remote repositories. ''' - # TODO: When BuildBranches become more context aware, if there - # are no uncommitted changes and the local versions are pushed - # we can skip pushing even if push_temporary is set. - # No uncommitted changes isn't sufficient reason to push the - # current HEAD - if self._push_temporary: - with morphlib.branchmanager.RemoteRefManager(False) as rrm: - for gd, (build_ref, index) in self._to_push.iteritems(): - remote = gd.get_remote('origin') - refspec = morphlib.gitdir.RefSpec(build_ref) - push_cb(gd=gd, build_ref=build_ref, - remote=remote, refspec=refspec) - rrm.push(remote, refspec) - self._register_cleanup(rrm.close) + with morphlib.branchmanager.RemoteRefManager(False) as rrm: + for gd, (build_ref, index) in self._to_push.iteritems(): + remote = gd.get_remote('origin') + refspec = morphlib.gitdir.RefSpec(build_ref) + push_cb(gd=gd, build_ref=build_ref, + remote=remote, refspec=refspec) + rrm.push(remote, refspec) + self._register_cleanup(rrm.close) @property def root_repo_url(self): '''URI of the repository that systems may be found in.''' - # TODO: When BuildBranches become more context aware, we only - # have to use the file:// URI when there's uncommitted changes - # and we can't push; or HEAD is not pushed and we can't push. - # All other times we can use the pushed branch - return (self._sb.get_config('branch.root') if self._push_temporary - else urlparse.urljoin('file://', self._root.dirname)) + return self._sb.get_config('branch.root') @property def root_ref(self): + return self._sb.get_config('branch.name') + + @property + def root_local_repo_url(self): + return urlparse.urljoin('file://', self._root.dirname) + + @property + def root_build_ref(self): '''Name of the ref of the repository that systems may be found in.''' - # TODO: When BuildBranches become more context aware, this can be - # HEAD when there's no uncommitted changes and we're not pushing; - # or we are pushing and there's no uncommitted changes and HEAD - # has been pushed. build_ref, index = self._to_push[self._root] return build_ref @@ -260,3 +280,47 @@ class BuildBranch(object): exceptions.append(e) if exceptions: raise BuildBranchCleanupError(self, exceptions) + + +@contextlib.contextmanager +def pushed_build_branch(bb, loader, changes_need_pushing, name, email, + build_uuid, status): + with contextlib.closing(bb) as bb: + def report_add(gd, build_ref, changed): + status(msg='Adding uncommitted changes '\ + 'in %(dirname)s to %(ref)s', + dirname=gd.dirname, ref=build_ref, chatty=True) + changes_made = bb.add_uncommitted_changes(add_cb=report_add) + unpushed = any(bb.get_unpushed_branches()) + + if not changes_made and not unpushed: + yield bb.root_repo_url, bb.root_ref + return + + def report_inject(gd): + status(msg='Injecting temporary build refs '\ + 'into morphologies in %(dirname)s', + dirname=gd.dirname, chatty=True) + bb.inject_build_refs(loader=loader, + use_local_repos=not changes_need_pushing, + inject_cb=report_inject) + + def report_commit(gd, build_ref): + status(msg='Committing changes in %(dirname)s '\ + 'to %(ref)s', + dirname=gd.dirname, ref=build_ref, + chatty=True) + bb.update_build_refs(name, email, build_uuid, + commit_cb=report_commit) + + if changes_need_pushing: + def report_push(gd, build_ref, remote, refspec): + status(msg='Pushing %(ref)s in %(dirname)s '\ + 'to %(remote)s', + ref=build_ref, dirname=gd.dirname, + remote=remote.get_push_url(), chatty=True) + bb.push_build_branches(push_cb=report_push) + + yield bb.root_repo_url, bb.root_build_ref + else: + yield bb.root_local_repo_url, bb.root_build_ref diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 5b0693cb..3966a0f0 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -282,6 +282,16 @@ class Remote(object): return self._get_remote_url(self.name, 'push') @staticmethod + def _parse_ls_remote_output(output): # pragma: no cover + for line in output.splitlines(): + sha1, refname = line.split(None, 1) + yield sha1, refname + + def ls(self): # pragma: no cover + out = self.gd._runcmd(['git', 'ls-remote', self.get_fetch_url()]) + return self._parse_ls_remote_output(out) + + @staticmethod def _parse_push_output(output): for line in output.splitlines(): m = PUSH_FORMAT.match(line) @@ -484,6 +494,10 @@ class GitDirectory(object): except cliapp.AppException as e: raise InvalidRefError(self, ref) + def disambiguate_ref(self, ref): # pragma: no cover + out = self._runcmd(['git', 'rev-parse', '--symbolic-full-name', ref]) + return out.strip() + def resolve_ref_to_commit(self, ref): return self._rev_parse('%s^{commit}' % ref) diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 3a489a61..64630c2b 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -184,35 +184,10 @@ class BuildPlugin(cliapp.Plugin): system=system_filename, branch=sb.system_branch_name) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, - push_temporary=push) - with contextlib.closing(bb) as bb: - def report_add(gd, build_ref, changed): - self.app.status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - bb.add_uncommitted_changes(add_cb=report_add) - - def report_inject(gd): - self.app.status(msg='Injecting temporary build refs '\ - 'into morphologies in %(dirname)s', - dirname=gd.dirname, chatty=True) - bb.inject_build_refs(loader, inject_cb=report_inject) - - def report_commit(gd, build_ref): - self.app.status(msg='Committing changes in %(dirname)s '\ - 'to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - bb.update_build_refs(name, email, build_uuid, - commit_cb=report_commit) - - def report_push(gd, build_ref, remote, refspec): - self.app.status(msg='Pushing %(ref)s in %(dirname)s '\ - 'to %(remote)s', - ref=build_ref, dirname=gd.dirname, - remote=remote.get_push_url(), chatty=True) - bb.push_build_branches(push_cb=report_push) - - build_command.build([bb.root_repo_url, - bb.root_ref, - system_filename]) + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=push, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, ref): + build_command.build([repo, ref, system_filename]) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 9f2d8eba..61b8145e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -319,44 +319,21 @@ class DeployPlugin(cliapp.Plugin): self.validate_deployment_options( env_vars, all_deployments, all_subsystems) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, - push_temporary=False) - with contextlib.closing(bb) as bb: - def report_add(gd, build_ref, changed): - self.app.status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - bb.add_uncommitted_changes(add_cb=report_add) - - def report_inject(gd): - self.app.status(msg='Injecting temporary build refs '\ - 'into morphologies in %(dirname)s', - dirname=gd.dirname, chatty=True) - bb.inject_build_refs(loader, inject_cb=report_inject) - - def report_commit(gd, build_ref): - self.app.status(msg='Committing changes in %(dirname)s '\ - 'to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - bb.update_build_refs(name, email, build_uuid, - commit_cb=report_commit) - - def report_push(gd, build_ref, remote, refspec): - self.app.status(msg='Pushing %(ref)s in %(dirname)s '\ - 'to %(remote)s', - ref=build_ref, dirname=gd.dirname, - remote=remote.get_push_url(), chatty=True) - bb.push_build_branches(push_cb=report_push) - + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=False, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, ref): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, bb.root_repo_url, - bb.root_ref, system, env_vars, - deployments, parent_location='') + root_repo_dir, repo, ref, system, + env_vars, deployments, + parent_location='') finally: shutil.rmtree(deploy_tempdir) |