summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2013-11-27 14:33:22 +0000
committerRichard Maw <richard.maw@codethink.co.uk>2013-11-29 16:11:44 +0000
commit198b59056058be476cb95e44fbe839f09258527c (patch)
treef2e2ca45d0aa23d6b54ec717575f4cc9838758af
parent3eb2b658b1f3a612c78c14f3d2cba2a1f0b1333f (diff)
downloadmorph-198b59056058be476cb95e44fbe839f09258527c.tar.gz
morphlib: Add BuildBranch abstraction
This is an abstraction on top of SystemBranchDirectories, providing the ability to add uncommitted changes to the temporary build branch, push temporary build branches and retrieve the correct repository URI and ref to build the system.
-rw-r--r--morphlib/__init__.py1
-rw-r--r--morphlib/buildbranch.py260
-rw-r--r--without-test-modules2
3 files changed, 263 insertions, 0 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index ad90bd4d..8f39cb30 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -50,6 +50,7 @@ import artifactcachereference
import artifactresolver
import branchmanager
import bins
+import buildbranch
import buildcommand
import buildenvironment
import buildsystem
diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py
new file mode 100644
index 00000000..d4426afb
--- /dev/null
+++ b/morphlib/buildbranch.py
@@ -0,0 +1,260 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import collections
+import os
+import urlparse
+
+import cliapp
+import fs.tempfs
+
+import morphlib
+
+
+class BuildBranchCleanupError(cliapp.AppException):
+ def __init__(self, bb, exceptions):
+ self.bb = bb
+ self.exceptions = exceptions
+ ex_nr = len(exceptions)
+ cliapp.AppException.__init__(
+ self, '%(ex_nr)d exceptions caught when cleaning up build branch'
+ % locals())
+
+
+class BuildBranch(object):
+ '''Represent the sources modified in a system branch.
+
+ This is an abstraction on top of SystemBranchDirectories, providing
+ the ability to add uncommitted changes to the temporary build branch,
+ push temporary build branches and retrieve the correct repository
+ URI and ref to build the system.
+
+ '''
+
+ # TODO: This currently always uses the temporary build ref. It
+ # would be better to not use local repositories and temporary refs,
+ # so building from a workspace appears to be identical to using
+ # `morph build-morphology`
+ def __init__(self, sb, build_ref_prefix, push_temporary):
+
+ self._sb = sb
+ self._push_temporary = push_temporary
+
+ self._cleanup = collections.deque()
+ self._to_push = {}
+ self._td = fs.tempfs.TempFS()
+ self._register_cleanup(self._td.close)
+
+ self._branch_root = sb.get_config('branch.root')
+ branch_uuid = sb.get_config('branch.uuid')
+
+ for gd in sb.list_git_directories():
+ try:
+ repo_uuid = gd.get_config('morph.uuid')
+ except cliapp.AppException:
+ # Not a repository cloned by morph, ignore
+ break
+ build_ref = os.path.join('refs/heads', build_ref_prefix,
+ branch_uuid, repo_uuid)
+ # index is commit of workspace + uncommitted changes may want
+ # to change to use user's index instead of user's commit,
+ # so they can add new files first
+ index = gd.get_index(self._td.getsyspath(repo_uuid))
+ index.set_to_tree(gd.resolve_ref_to_tree(gd.HEAD))
+ self._to_push[gd] = (build_ref, index)
+
+ rootinfo, = ((gd, index) for gd, (build_ref, index)
+ in self._to_push.iteritems()
+ if gd.get_config('morph.repository') == self._branch_root)
+ self._root, self._root_index = rootinfo
+
+ def _register_cleanup(self, func, *args, **kwargs):
+ self._cleanup.append((func, args, kwargs))
+
+ def add_uncommitted_changes(self):
+ '''Add any uncommitted changes to temporary build GitIndexes'''
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ changed = [to_path for code, to_path, from_path
+ in index.get_uncommitted_changes()]
+ if not changed:
+ continue
+ yield gd, build_ref
+ index.add_files_from_working_tree(changed)
+
+ @staticmethod
+ def _hash_morphologies(gd, morphologies, loader):
+ '''Hash morphologies and return object info'''
+ for morphology in morphologies:
+ loader.unset_defaults(morphology)
+ sha1 = gd.store_blob(loader.save_to_string(morphology))
+ yield 0100644, sha1, morphology.filename
+
+ def inject_build_refs(self, loader):
+ '''Update system and stratum morphologies to point to our branch.
+
+ For all edited repositories, this alter the temporary GitIndex
+ of the morphs repositories to point their temporary build branch
+ versions.
+
+ `loader` is a MorphologyLoader that is used to convert morphology
+ files into their in-memory representations and back again.
+
+ '''
+ root_repo = self._root.get_config('morph.repository')
+ root_ref = self._root.HEAD
+ morphs = morphlib.morphset.MorphologySet()
+ for morph in self._sb.load_all_morphologies(loader):
+ morphs.add_morphology(morph)
+
+ sb_info = {}
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ repo, ref = gd.get_config('morph.repository'), gd.HEAD
+ sb_info[repo, ref] = (gd, build_ref)
+
+ def filter(m, kind, spec):
+ return (spec['repo'], spec['ref']) in sb_info
+ def process(m, kind, spec):
+ repo, ref = spec['repo'], spec['ref']
+ gd, build_ref = sb_info[repo, ref]
+ if (repo, ref) == (root_repo, root_ref):
+ spec['repo'] = None
+ spec['ref'] = None
+ return True
+ if not self._push_temporary:
+ spec['repo'] = urlparse.urljoin('file://', gd.dirname)
+ spec['ref'] = build_ref
+ return True
+
+ morphs.traverse_specs(process, filter)
+
+ if any(m.dirty for m in morphs.morphologies):
+ yield self._root
+
+ self._root_index.add_files_from_index_info(
+ self._hash_morphologies(self._root, morphs.morphologies, loader))
+
+ def update_build_refs(self, name, email, uuid):
+ '''Commit changes in temporary GitIndexes to temporary branches.
+
+ `name` and `email` are required to construct the commit author info.
+ `uuid` is used to identify each build uniquely and is included
+ in the commit message.
+
+ A new commit is added to the temporary build branch of each of
+ the repositories in the SystemBranch with:
+ 1. The tree of anything currently in the temporary GitIndex.
+ This is the same as the current commit on HEAD unless
+ `add_uncommitted_changes` or `inject_build_refs` have
+ been called.
+ 2. the parent of the previous temporary commit, or the last
+ commit of the working tree if there has been no previous
+ commits
+ 3. author and committer email as specified by `email`, author
+ name of `name` and committer name of 'Morph (on behalf of
+ `name`)'
+ 4. commit message describing the current build using `uuid`
+
+ '''
+ commit_message = 'Morph build %s\n\nSystem branch: %s\n' % \
+ (uuid, self._sb.system_branch_name)
+ author_name = name
+ committer_name = 'Morph (on behalf of %s)' % name
+ author_email = committer_email = email
+
+ with morphlib.branchmanager.LocalRefManager() as lrm:
+ for gd, (build_ref, index) in self._to_push.iteritems():
+ yield gd, build_ref
+ tree = index.write_tree()
+ try:
+ parent = gd.resolve_ref_to_commit(build_ref)
+ except morphlib.gitdir.InvalidRefError:
+ parent = gd.resolve_ref_to_commit(gd.HEAD)
+
+ commit = gd.commit_tree(tree, parent=parent,
+ committer_name=committer_name,
+ committer_email=committer_email,
+ author_name=author_name,
+ author_email=author_email,
+ message=commit_message)
+ try:
+ old_commit = gd.resolve_ref_to_commit(build_ref)
+ except morphlib.gitdir.InvalidRefError:
+ lrm.add(gd, build_ref, commit)
+ else:
+ # NOTE: This will fail if build_ref pointed to a tag,
+ # due to resolve_ref_to_commit returning the
+ # commit id of tags, but since it's only morph
+ # that touches those refs, it should not be
+ # a problem.
+ lrm.update(gd, build_ref, commit, old_commit)
+
+ def push_build_branches(self):
+ '''Push all temporary build branches to the remote repositories.
+
+ This is a no-op if the BuildBranch was constructed with
+ `push_temporary` as False, so that the code flow for the user of
+ the BuildBranch can be the same when it can be pushed as when
+ it can't.
+
+ '''
+ # TODO: When BuildBranches become more context aware, if there
+ # are no uncommitted changes and the local versions are pushed
+ # we can skip pushing even if push_temporary is set.
+ # No uncommitted changes isn't sufficient reason to push the
+ # current HEAD
+ if self._push_temporary:
+ with morphlib.branchmanager.RemoteRefManager(False) as rrm:
+ for gd, build_ref in self._to_push.iterkeys():
+ remote = gd.get_remote('origin')
+ yield gd, build_ref, remote
+ refspec = morphlib.gitdir.RefSpec(build_ref)
+ rrm.push(remote, refspec)
+ self._register_cleanup(rrm.close)
+
+ @property
+ def root_repo_url(self):
+ '''URI of the repository that systems may be found in.'''
+ # TODO: When BuildBranches become more context aware, we only
+ # have to use the file:// URI when there's uncommitted changes
+ # and we can't push; or HEAD is not pushed and we can't push.
+ # All other times we can use the pushed branch
+ return (self._sb.get_config('branch.root') if self._push_temporary
+ else urlparse.urljoin('file://', self._root.dirname))
+
+ @property
+ def root_ref(self):
+ '''Name of the ref of the repository that systems may be found in.'''
+ # TODO: When BuildBranches become more context aware, this can be
+ # HEAD when there's no uncommitted changes and we're not pushing;
+ # or we are pushing and there's no uncommitted changes and HEAD
+ # has been pushed.
+ build_ref, index = self._to_push[self._root]
+ return build_ref
+
+ def close(self):
+ '''Clean up any resources acquired during operation.'''
+ # TODO: This is a common pattern for our context managers,
+ # we could do with a way to share the common code. I suggest the
+ # ExitStack from python3.4 or the contextlib2 module.
+ exceptions = []
+ while self._cleanup:
+ func, args, kwargs = self._cleanup.pop()
+ try:
+ func(*args, **kwargs)
+ except Exception, e:
+ exceptions.append(e)
+ if exceptions:
+ raise BuildBranchCleanupError(self, exceptions)
diff --git a/without-test-modules b/without-test-modules
index 4efcdb40..c34ba59c 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -28,3 +28,5 @@ morphlib/plugins/trovectl_plugin.py
morphlib/plugins/gc_plugin.py
morphlib/plugins/branch_and_merge_new_plugin.py
morphlib/plugins/print_architecture_plugin.py
+# Not unit tested, since it needs a full system branch
+morphlib/buildbranch.py