summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2014-08-08 23:05:13 +0100
committerRichard Maw <richard.maw@codethink.co.uk>2014-08-12 11:00:29 +0100
commit6c9a5d65d615abf75e640f5ceda332ef42854d57 (patch)
tree382deb1ed959ed5f9e682bacd13b198ba5a95388 /morphlib
parent54e03f2e5005775c8e8c434094256feccd91c488 (diff)
downloadmorph-6c9a5d65d615abf75e640f5ceda332ef42854d57.tar.gz
Avoid creating and pushing temporary build branches when they aren't necessary.
Sorry about the big lump, I can split it into a nicer set of changes, but they didn't naturally emerge in a nice series. This creates a pushed_build_branch context manager, to eliminate code duplication between build and deploy. Rather than the build branch being constructed knowing whether it needs to push the branch, it infers that from the state of the repositories, and whether a local build would be possible. If there are no uncommitted changes and all local branches are pushed, then it doesn't create temporary branches or push them, and instead uses what it already has. It will currently create and use temporary build branches even for chunks that have no local changes, but it's pretty cheap, and doesn't require re-working the build-ref injection code to check whether there are local changes.
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/buildbranch.py134
-rw-r--r--morphlib/gitdir.py14
-rw-r--r--morphlib/plugins/build_plugin.py39
-rw-r--r--morphlib/plugins/deploy_plugin.py41
4 files changed, 129 insertions, 99 deletions
diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py
index 7aa75dc1..885f5cf8 100644
--- a/morphlib/buildbranch.py
+++ b/morphlib/buildbranch.py
@@ -15,6 +15,7 @@
import collections
+import contextlib
import os
import urlparse
@@ -48,10 +49,9 @@ class BuildBranch(object):
# would be better to not use local repositories and temporary refs,
# so building from a workspace appears to be identical to using
# `morph build-morphology`
- def __init__(self, sb, build_ref_prefix, push_temporary):
+ def __init__(self, sb, build_ref_prefix):
self._sb = sb
- self._push_temporary = push_temporary
self._cleanup = collections.deque()
self._to_push = {}
@@ -86,13 +86,16 @@ class BuildBranch(object):
def add_uncommitted_changes(self, add_cb=lambda **kwargs: None):
'''Add any uncommitted changes to temporary build GitIndexes'''
+ changes_made = False
for gd, (build_ref, index) in self._to_push.iteritems():
changed = [to_path for code, to_path, from_path
in index.get_uncommitted_changes()]
if not changed:
continue
add_cb(gd=gd, build_ref=gd, changed=changed)
+ changes_made = True
index.add_files_from_working_tree(changed)
+ return changes_made
@staticmethod
def _hash_morphologies(gd, morphologies, loader):
@@ -102,7 +105,8 @@ class BuildBranch(object):
sha1 = gd.store_blob(loader.save_to_string(morphology))
yield 0100644, sha1, morphology.filename
- def inject_build_refs(self, loader, inject_cb=lambda **kwargs: None):
+ def inject_build_refs(self, loader, use_local_repos,
+ inject_cb=lambda **kwargs: None):
'''Update system and stratum morphologies to point to our branch.
For all edited repositories, this alter the temporary GitIndex
@@ -133,7 +137,7 @@ class BuildBranch(object):
spec['repo'] = None
spec['ref'] = None
return True
- if not self._push_temporary:
+ if use_local_repos:
spec['repo'] = urlparse.urljoin('file://', gd.dirname)
spec['ref'] = build_ref
return True
@@ -143,6 +147,8 @@ class BuildBranch(object):
if any(m.dirty for m in morphs.morphologies):
inject_cb(gd=self._root)
+ # TODO: Prevent it hashing unchanged morphologies, while still
+ # hashing uncommitted ones.
self._root_index.add_files_from_index_info(
self._hash_morphologies(self._root, morphs.morphologies, loader))
@@ -177,12 +183,18 @@ class BuildBranch(object):
with morphlib.branchmanager.LocalRefManager() as lrm:
for gd, (build_ref, index) in self._to_push.iteritems():
- commit_cb(gd=gd, build_ref=build_ref)
tree = index.write_tree()
try:
parent = gd.resolve_ref_to_commit(build_ref)
except morphlib.gitdir.InvalidRefError:
parent = gd.resolve_ref_to_commit(gd.HEAD)
+ else:
+ # Skip updating ref if we already have a temporary
+ # build branch and have this tree on the branch
+ if tree == gd.resolve_ref_to_tree(build_ref):
+ continue
+
+ commit_cb(gd=gd, build_ref=build_ref)
commit = gd.commit_tree(tree, parent=parent,
committer_name=committer_name,
@@ -202,47 +214,55 @@ class BuildBranch(object):
# a problem.
lrm.update(gd, build_ref, commit, old_commit)
- def push_build_branches(self, push_cb=lambda **kwargs: None):
- '''Push all temporary build branches to the remote repositories.
+ def get_unpushed_branches(self):
+ '''Work out which, if any, local branches need to be pushed to build
+
+ NOTE: This assumes that the refs in the morphologies and the
+ refs in the local checkouts match.
- This is a no-op if the BuildBranch was constructed with
- `push_temporary` as False, so that the code flow for the user of
- the BuildBranch can be the same when it can be pushed as when
- it can't.
+ '''
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ remote = gd.get_remote('origin')
+ head_ref = gd.disambiguate_ref(gd.HEAD)
+ head_sha1 = gd.resolve_ref_to_commit(head_ref)
+ pushed_refs = sorted(
+ (remote_ref
+ for remote_sha1, remote_ref in remote.ls()
+ # substring match of refs, since ref may be a tag,
+ # in which case it would end with ^{}
+ if remote_sha1 == head_sha1 and head_ref in remote_ref),
+ key=len)
+ if not pushed_refs:
+ yield gd
+ def push_build_branches(self, push_cb=lambda **kwargs: None):
+ '''Push all temporary build branches to the remote repositories.
'''
- # TODO: When BuildBranches become more context aware, if there
- # are no uncommitted changes and the local versions are pushed
- # we can skip pushing even if push_temporary is set.
- # No uncommitted changes isn't sufficient reason to push the
- # current HEAD
- if self._push_temporary:
- with morphlib.branchmanager.RemoteRefManager(False) as rrm:
- for gd, (build_ref, index) in self._to_push.iteritems():
- remote = gd.get_remote('origin')
- refspec = morphlib.gitdir.RefSpec(build_ref)
- push_cb(gd=gd, build_ref=build_ref,
- remote=remote, refspec=refspec)
- rrm.push(remote, refspec)
- self._register_cleanup(rrm.close)
+ with morphlib.branchmanager.RemoteRefManager(False) as rrm:
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ remote = gd.get_remote('origin')
+ refspec = morphlib.gitdir.RefSpec(build_ref)
+ push_cb(gd=gd, build_ref=build_ref,
+ remote=remote, refspec=refspec)
+ rrm.push(remote, refspec)
+ self._register_cleanup(rrm.close)
@property
def root_repo_url(self):
'''URI of the repository that systems may be found in.'''
- # TODO: When BuildBranches become more context aware, we only
- # have to use the file:// URI when there's uncommitted changes
- # and we can't push; or HEAD is not pushed and we can't push.
- # All other times we can use the pushed branch
- return (self._sb.get_config('branch.root') if self._push_temporary
- else urlparse.urljoin('file://', self._root.dirname))
+ return self._sb.get_config('branch.root')
@property
def root_ref(self):
+ return self._sb.get_config('branch.name')
+
+ @property
+ def root_local_repo_url(self):
+ return urlparse.urljoin('file://', self._root.dirname)
+
+ @property
+ def root_build_ref(self):
'''Name of the ref of the repository that systems may be found in.'''
- # TODO: When BuildBranches become more context aware, this can be
- # HEAD when there's no uncommitted changes and we're not pushing;
- # or we are pushing and there's no uncommitted changes and HEAD
- # has been pushed.
build_ref, index = self._to_push[self._root]
return build_ref
@@ -260,3 +280,47 @@ class BuildBranch(object):
exceptions.append(e)
if exceptions:
raise BuildBranchCleanupError(self, exceptions)
+
+
+@contextlib.contextmanager
+def pushed_build_branch(bb, loader, changes_need_pushing, name, email,
+ build_uuid, status):
+ with contextlib.closing(bb) as bb:
+ def report_add(gd, build_ref, changed):
+ status(msg='Adding uncommitted changes '\
+ 'in %(dirname)s to %(ref)s',
+ dirname=gd.dirname, ref=build_ref, chatty=True)
+ changes_made = bb.add_uncommitted_changes(add_cb=report_add)
+ unpushed = any(bb.get_unpushed_branches())
+
+ if not changes_made and not unpushed:
+ yield bb.root_repo_url, bb.root_ref
+ return
+
+ def report_inject(gd):
+ status(msg='Injecting temporary build refs '\
+ 'into morphologies in %(dirname)s',
+ dirname=gd.dirname, chatty=True)
+ bb.inject_build_refs(loader=loader,
+ use_local_repos=not changes_need_pushing,
+ inject_cb=report_inject)
+
+ def report_commit(gd, build_ref):
+ status(msg='Committing changes in %(dirname)s '\
+ 'to %(ref)s',
+ dirname=gd.dirname, ref=build_ref,
+ chatty=True)
+ bb.update_build_refs(name, email, build_uuid,
+ commit_cb=report_commit)
+
+ if changes_need_pushing:
+ def report_push(gd, build_ref, remote, refspec):
+ status(msg='Pushing %(ref)s in %(dirname)s '\
+ 'to %(remote)s',
+ ref=build_ref, dirname=gd.dirname,
+ remote=remote.get_push_url(), chatty=True)
+ bb.push_build_branches(push_cb=report_push)
+
+ yield bb.root_repo_url, bb.root_build_ref
+ else:
+ yield bb.root_local_repo_url, bb.root_build_ref
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 5b0693cb..3966a0f0 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -282,6 +282,16 @@ class Remote(object):
return self._get_remote_url(self.name, 'push')
@staticmethod
+ def _parse_ls_remote_output(output): # pragma: no cover
+ for line in output.splitlines():
+ sha1, refname = line.split(None, 1)
+ yield sha1, refname
+
+ def ls(self): # pragma: no cover
+ out = self.gd._runcmd(['git', 'ls-remote', self.get_fetch_url()])
+ return self._parse_ls_remote_output(out)
+
+ @staticmethod
def _parse_push_output(output):
for line in output.splitlines():
m = PUSH_FORMAT.match(line)
@@ -484,6 +494,10 @@ class GitDirectory(object):
except cliapp.AppException as e:
raise InvalidRefError(self, ref)
+ def disambiguate_ref(self, ref): # pragma: no cover
+ out = self._runcmd(['git', 'rev-parse', '--symbolic-full-name', ref])
+ return out.strip()
+
def resolve_ref_to_commit(self, ref):
return self._rev_parse('%s^{commit}' % ref)
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 3a489a61..64630c2b 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -184,35 +184,10 @@ class BuildPlugin(cliapp.Plugin):
system=system_filename,
branch=sb.system_branch_name)
- bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix,
- push_temporary=push)
- with contextlib.closing(bb) as bb:
- def report_add(gd, build_ref, changed):
- self.app.status(msg='Adding uncommitted changes '\
- 'in %(dirname)s to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.add_uncommitted_changes(add_cb=report_add)
-
- def report_inject(gd):
- self.app.status(msg='Injecting temporary build refs '\
- 'into morphologies in %(dirname)s',
- dirname=gd.dirname, chatty=True)
- bb.inject_build_refs(loader, inject_cb=report_inject)
-
- def report_commit(gd, build_ref):
- self.app.status(msg='Committing changes in %(dirname)s '\
- 'to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.update_build_refs(name, email, build_uuid,
- commit_cb=report_commit)
-
- def report_push(gd, build_ref, remote, refspec):
- self.app.status(msg='Pushing %(ref)s in %(dirname)s '\
- 'to %(remote)s',
- ref=build_ref, dirname=gd.dirname,
- remote=remote.get_push_url(), chatty=True)
- bb.push_build_branches(push_cb=report_push)
-
- build_command.build([bb.root_repo_url,
- bb.root_ref,
- system_filename])
+ bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix)
+ pbb = morphlib.buildbranch.pushed_build_branch(
+ bb, loader=loader, changes_need_pushing=push,
+ name=name, email=email, build_uuid=build_uuid,
+ status=self.app.status)
+ with pbb as (repo, ref):
+ build_command.build([repo, ref, system_filename])
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 9f2d8eba..61b8145e 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -319,44 +319,21 @@ class DeployPlugin(cliapp.Plugin):
self.validate_deployment_options(
env_vars, all_deployments, all_subsystems)
- bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix,
- push_temporary=False)
- with contextlib.closing(bb) as bb:
- def report_add(gd, build_ref, changed):
- self.app.status(msg='Adding uncommitted changes '\
- 'in %(dirname)s to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.add_uncommitted_changes(add_cb=report_add)
-
- def report_inject(gd):
- self.app.status(msg='Injecting temporary build refs '\
- 'into morphologies in %(dirname)s',
- dirname=gd.dirname, chatty=True)
- bb.inject_build_refs(loader, inject_cb=report_inject)
-
- def report_commit(gd, build_ref):
- self.app.status(msg='Committing changes in %(dirname)s '\
- 'to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.update_build_refs(name, email, build_uuid,
- commit_cb=report_commit)
-
- def report_push(gd, build_ref, remote, refspec):
- self.app.status(msg='Pushing %(ref)s in %(dirname)s '\
- 'to %(remote)s',
- ref=build_ref, dirname=gd.dirname,
- remote=remote.get_push_url(), chatty=True)
- bb.push_build_branches(push_cb=report_push)
-
+ bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix)
+ pbb = morphlib.buildbranch.pushed_build_branch(
+ bb, loader=loader, changes_need_pushing=False,
+ name=name, email=email, build_uuid=build_uuid,
+ status=self.app.status)
+ with pbb as (repo, ref):
# Create a tempdir for this deployment to work in
deploy_tempdir = tempfile.mkdtemp(
dir=os.path.join(self.app.settings['tempdir'], 'deployments'))
try:
for system in cluster_morphology['systems']:
self.deploy_system(build_command, deploy_tempdir,
- root_repo_dir, bb.root_repo_url,
- bb.root_ref, system, env_vars,
- deployments, parent_location='')
+ root_repo_dir, repo, ref, system,
+ env_vars, deployments,
+ parent_location='')
finally:
shutil.rmtree(deploy_tempdir)