summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--morphlib/buildbranch.py134
-rw-r--r--morphlib/gitdir.py14
-rw-r--r--morphlib/plugins/build_plugin.py39
-rw-r--r--morphlib/plugins/deploy_plugin.py41
4 files changed, 129 insertions, 99 deletions
diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py
index 7aa75dc1..885f5cf8 100644
--- a/morphlib/buildbranch.py
+++ b/morphlib/buildbranch.py
@@ -15,6 +15,7 @@
import collections
+import contextlib
import os
import urlparse
@@ -48,10 +49,9 @@ class BuildBranch(object):
# would be better to not use local repositories and temporary refs,
# so building from a workspace appears to be identical to using
# `morph build-morphology`
- def __init__(self, sb, build_ref_prefix, push_temporary):
+ def __init__(self, sb, build_ref_prefix):
self._sb = sb
- self._push_temporary = push_temporary
self._cleanup = collections.deque()
self._to_push = {}
@@ -86,13 +86,16 @@ class BuildBranch(object):
def add_uncommitted_changes(self, add_cb=lambda **kwargs: None):
'''Add any uncommitted changes to temporary build GitIndexes'''
+ changes_made = False
for gd, (build_ref, index) in self._to_push.iteritems():
changed = [to_path for code, to_path, from_path
in index.get_uncommitted_changes()]
if not changed:
continue
add_cb(gd=gd, build_ref=gd, changed=changed)
+ changes_made = True
index.add_files_from_working_tree(changed)
+ return changes_made
@staticmethod
def _hash_morphologies(gd, morphologies, loader):
@@ -102,7 +105,8 @@ class BuildBranch(object):
sha1 = gd.store_blob(loader.save_to_string(morphology))
yield 0100644, sha1, morphology.filename
- def inject_build_refs(self, loader, inject_cb=lambda **kwargs: None):
+ def inject_build_refs(self, loader, use_local_repos,
+ inject_cb=lambda **kwargs: None):
'''Update system and stratum morphologies to point to our branch.
For all edited repositories, this alter the temporary GitIndex
@@ -133,7 +137,7 @@ class BuildBranch(object):
spec['repo'] = None
spec['ref'] = None
return True
- if not self._push_temporary:
+ if use_local_repos:
spec['repo'] = urlparse.urljoin('file://', gd.dirname)
spec['ref'] = build_ref
return True
@@ -143,6 +147,8 @@ class BuildBranch(object):
if any(m.dirty for m in morphs.morphologies):
inject_cb(gd=self._root)
+ # TODO: Prevent it hashing unchanged morphologies, while still
+ # hashing uncommitted ones.
self._root_index.add_files_from_index_info(
self._hash_morphologies(self._root, morphs.morphologies, loader))
@@ -177,12 +183,18 @@ class BuildBranch(object):
with morphlib.branchmanager.LocalRefManager() as lrm:
for gd, (build_ref, index) in self._to_push.iteritems():
- commit_cb(gd=gd, build_ref=build_ref)
tree = index.write_tree()
try:
parent = gd.resolve_ref_to_commit(build_ref)
except morphlib.gitdir.InvalidRefError:
parent = gd.resolve_ref_to_commit(gd.HEAD)
+ else:
+ # Skip updating ref if we already have a temporary
+ # build branch and have this tree on the branch
+ if tree == gd.resolve_ref_to_tree(build_ref):
+ continue
+
+ commit_cb(gd=gd, build_ref=build_ref)
commit = gd.commit_tree(tree, parent=parent,
committer_name=committer_name,
@@ -202,47 +214,55 @@ class BuildBranch(object):
# a problem.
lrm.update(gd, build_ref, commit, old_commit)
- def push_build_branches(self, push_cb=lambda **kwargs: None):
- '''Push all temporary build branches to the remote repositories.
+ def get_unpushed_branches(self):
+ '''Work out which, if any, local branches need to be pushed to build
+
+ NOTE: This assumes that the refs in the morphologies and the
+ refs in the local checkouts match.
- This is a no-op if the BuildBranch was constructed with
- `push_temporary` as False, so that the code flow for the user of
- the BuildBranch can be the same when it can be pushed as when
- it can't.
+ '''
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ remote = gd.get_remote('origin')
+ head_ref = gd.disambiguate_ref(gd.HEAD)
+ head_sha1 = gd.resolve_ref_to_commit(head_ref)
+ pushed_refs = sorted(
+ (remote_ref
+ for remote_sha1, remote_ref in remote.ls()
+ # substring match of refs, since ref may be a tag,
+ # in which case it would end with ^{}
+ if remote_sha1 == head_sha1 and head_ref in remote_ref),
+ key=len)
+ if not pushed_refs:
+ yield gd
+ def push_build_branches(self, push_cb=lambda **kwargs: None):
+ '''Push all temporary build branches to the remote repositories.
'''
- # TODO: When BuildBranches become more context aware, if there
- # are no uncommitted changes and the local versions are pushed
- # we can skip pushing even if push_temporary is set.
- # No uncommitted changes isn't sufficient reason to push the
- # current HEAD
- if self._push_temporary:
- with morphlib.branchmanager.RemoteRefManager(False) as rrm:
- for gd, (build_ref, index) in self._to_push.iteritems():
- remote = gd.get_remote('origin')
- refspec = morphlib.gitdir.RefSpec(build_ref)
- push_cb(gd=gd, build_ref=build_ref,
- remote=remote, refspec=refspec)
- rrm.push(remote, refspec)
- self._register_cleanup(rrm.close)
+ with morphlib.branchmanager.RemoteRefManager(False) as rrm:
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ remote = gd.get_remote('origin')
+ refspec = morphlib.gitdir.RefSpec(build_ref)
+ push_cb(gd=gd, build_ref=build_ref,
+ remote=remote, refspec=refspec)
+ rrm.push(remote, refspec)
+ self._register_cleanup(rrm.close)
@property
def root_repo_url(self):
'''URI of the repository that systems may be found in.'''
- # TODO: When BuildBranches become more context aware, we only
- # have to use the file:// URI when there's uncommitted changes
- # and we can't push; or HEAD is not pushed and we can't push.
- # All other times we can use the pushed branch
- return (self._sb.get_config('branch.root') if self._push_temporary
- else urlparse.urljoin('file://', self._root.dirname))
+ return self._sb.get_config('branch.root')
@property
def root_ref(self):
+ return self._sb.get_config('branch.name')
+
+ @property
+ def root_local_repo_url(self):
+ return urlparse.urljoin('file://', self._root.dirname)
+
+ @property
+ def root_build_ref(self):
'''Name of the ref of the repository that systems may be found in.'''
- # TODO: When BuildBranches become more context aware, this can be
- # HEAD when there's no uncommitted changes and we're not pushing;
- # or we are pushing and there's no uncommitted changes and HEAD
- # has been pushed.
build_ref, index = self._to_push[self._root]
return build_ref
@@ -260,3 +280,47 @@ class BuildBranch(object):
exceptions.append(e)
if exceptions:
raise BuildBranchCleanupError(self, exceptions)
+
+
+@contextlib.contextmanager
+def pushed_build_branch(bb, loader, changes_need_pushing, name, email,
+ build_uuid, status):
+ with contextlib.closing(bb) as bb:
+ def report_add(gd, build_ref, changed):
+ status(msg='Adding uncommitted changes '\
+ 'in %(dirname)s to %(ref)s',
+ dirname=gd.dirname, ref=build_ref, chatty=True)
+ changes_made = bb.add_uncommitted_changes(add_cb=report_add)
+ unpushed = any(bb.get_unpushed_branches())
+
+ if not changes_made and not unpushed:
+ yield bb.root_repo_url, bb.root_ref
+ return
+
+ def report_inject(gd):
+ status(msg='Injecting temporary build refs '\
+ 'into morphologies in %(dirname)s',
+ dirname=gd.dirname, chatty=True)
+ bb.inject_build_refs(loader=loader,
+ use_local_repos=not changes_need_pushing,
+ inject_cb=report_inject)
+
+ def report_commit(gd, build_ref):
+ status(msg='Committing changes in %(dirname)s '\
+ 'to %(ref)s',
+ dirname=gd.dirname, ref=build_ref,
+ chatty=True)
+ bb.update_build_refs(name, email, build_uuid,
+ commit_cb=report_commit)
+
+ if changes_need_pushing:
+ def report_push(gd, build_ref, remote, refspec):
+ status(msg='Pushing %(ref)s in %(dirname)s '\
+ 'to %(remote)s',
+ ref=build_ref, dirname=gd.dirname,
+ remote=remote.get_push_url(), chatty=True)
+ bb.push_build_branches(push_cb=report_push)
+
+ yield bb.root_repo_url, bb.root_build_ref
+ else:
+ yield bb.root_local_repo_url, bb.root_build_ref
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 5b0693cb..3966a0f0 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -282,6 +282,16 @@ class Remote(object):
return self._get_remote_url(self.name, 'push')
@staticmethod
+ def _parse_ls_remote_output(output): # pragma: no cover
+ for line in output.splitlines():
+ sha1, refname = line.split(None, 1)
+ yield sha1, refname
+
+ def ls(self): # pragma: no cover
+ out = self.gd._runcmd(['git', 'ls-remote', self.get_fetch_url()])
+ return self._parse_ls_remote_output(out)
+
+ @staticmethod
def _parse_push_output(output):
for line in output.splitlines():
m = PUSH_FORMAT.match(line)
@@ -484,6 +494,10 @@ class GitDirectory(object):
except cliapp.AppException as e:
raise InvalidRefError(self, ref)
+ def disambiguate_ref(self, ref): # pragma: no cover
+ out = self._runcmd(['git', 'rev-parse', '--symbolic-full-name', ref])
+ return out.strip()
+
def resolve_ref_to_commit(self, ref):
return self._rev_parse('%s^{commit}' % ref)
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 3a489a61..64630c2b 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -184,35 +184,10 @@ class BuildPlugin(cliapp.Plugin):
system=system_filename,
branch=sb.system_branch_name)
- bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix,
- push_temporary=push)
- with contextlib.closing(bb) as bb:
- def report_add(gd, build_ref, changed):
- self.app.status(msg='Adding uncommitted changes '\
- 'in %(dirname)s to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.add_uncommitted_changes(add_cb=report_add)
-
- def report_inject(gd):
- self.app.status(msg='Injecting temporary build refs '\
- 'into morphologies in %(dirname)s',
- dirname=gd.dirname, chatty=True)
- bb.inject_build_refs(loader, inject_cb=report_inject)
-
- def report_commit(gd, build_ref):
- self.app.status(msg='Committing changes in %(dirname)s '\
- 'to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.update_build_refs(name, email, build_uuid,
- commit_cb=report_commit)
-
- def report_push(gd, build_ref, remote, refspec):
- self.app.status(msg='Pushing %(ref)s in %(dirname)s '\
- 'to %(remote)s',
- ref=build_ref, dirname=gd.dirname,
- remote=remote.get_push_url(), chatty=True)
- bb.push_build_branches(push_cb=report_push)
-
- build_command.build([bb.root_repo_url,
- bb.root_ref,
- system_filename])
+ bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix)
+ pbb = morphlib.buildbranch.pushed_build_branch(
+ bb, loader=loader, changes_need_pushing=push,
+ name=name, email=email, build_uuid=build_uuid,
+ status=self.app.status)
+ with pbb as (repo, ref):
+ build_command.build([repo, ref, system_filename])
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 9f2d8eba..61b8145e 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -319,44 +319,21 @@ class DeployPlugin(cliapp.Plugin):
self.validate_deployment_options(
env_vars, all_deployments, all_subsystems)
- bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix,
- push_temporary=False)
- with contextlib.closing(bb) as bb:
- def report_add(gd, build_ref, changed):
- self.app.status(msg='Adding uncommitted changes '\
- 'in %(dirname)s to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.add_uncommitted_changes(add_cb=report_add)
-
- def report_inject(gd):
- self.app.status(msg='Injecting temporary build refs '\
- 'into morphologies in %(dirname)s',
- dirname=gd.dirname, chatty=True)
- bb.inject_build_refs(loader, inject_cb=report_inject)
-
- def report_commit(gd, build_ref):
- self.app.status(msg='Committing changes in %(dirname)s '\
- 'to %(ref)s',
- dirname=gd.dirname, ref=build_ref, chatty=True)
- bb.update_build_refs(name, email, build_uuid,
- commit_cb=report_commit)
-
- def report_push(gd, build_ref, remote, refspec):
- self.app.status(msg='Pushing %(ref)s in %(dirname)s '\
- 'to %(remote)s',
- ref=build_ref, dirname=gd.dirname,
- remote=remote.get_push_url(), chatty=True)
- bb.push_build_branches(push_cb=report_push)
-
+ bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix)
+ pbb = morphlib.buildbranch.pushed_build_branch(
+ bb, loader=loader, changes_need_pushing=False,
+ name=name, email=email, build_uuid=build_uuid,
+ status=self.app.status)
+ with pbb as (repo, ref):
# Create a tempdir for this deployment to work in
deploy_tempdir = tempfile.mkdtemp(
dir=os.path.join(self.app.settings['tempdir'], 'deployments'))
try:
for system in cluster_morphology['systems']:
self.deploy_system(build_command, deploy_tempdir,
- root_repo_dir, bb.root_repo_url,
- bb.root_ref, system, env_vars,
- deployments, parent_location='')
+ root_repo_dir, repo, ref, system,
+ env_vars, deployments,
+ parent_location='')
finally:
shutil.rmtree(deploy_tempdir)