diff options
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/buildbranch.py | 260 | ||||
| -rw-r--r-- | without-test-modules | 2 |
3 files changed, 263 insertions, 0 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index ad90bd4d..8f39cb30 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -50,6 +50,7 @@ import artifactcachereference import artifactresolver import branchmanager import bins +import buildbranch import buildcommand import buildenvironment import buildsystem diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py new file mode 100644 index 00000000..d4426afb --- /dev/null +++ b/morphlib/buildbranch.py @@ -0,0 +1,260 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import os +import urlparse + +import cliapp +import fs.tempfs + +import morphlib + + +class BuildBranchCleanupError(cliapp.AppException): + def __init__(self, bb, exceptions): + self.bb = bb + self.exceptions = exceptions + ex_nr = len(exceptions) + cliapp.AppException.__init__( + self, '%(ex_nr)d exceptions caught when cleaning up build branch' + % locals()) + + +class BuildBranch(object): + '''Represent the sources modified in a system branch. + + This is an abstraction on top of SystemBranchDirectories, providing + the ability to add uncommitted changes to the temporary build branch, + push temporary build branches and retrieve the correct repository + URI and ref to build the system. + + ''' + + # TODO: This currently always uses the temporary build ref. It + # would be better to not use local repositories and temporary refs, + # so building from a workspace appears to be identical to using + # `morph build-morphology` + def __init__(self, sb, build_ref_prefix, push_temporary): + + self._sb = sb + self._push_temporary = push_temporary + + self._cleanup = collections.deque() + self._to_push = {} + self._td = fs.tempfs.TempFS() + self._register_cleanup(self._td.close) + + self._branch_root = sb.get_config('branch.root') + branch_uuid = sb.get_config('branch.uuid') + + for gd in sb.list_git_directories(): + try: + repo_uuid = gd.get_config('morph.uuid') + except cliapp.AppException: + # Not a repository cloned by morph, ignore + break + build_ref = os.path.join('refs/heads', build_ref_prefix, + branch_uuid, repo_uuid) + # index is commit of workspace + uncommitted changes may want + # to change to use user's index instead of user's commit, + # so they can add new files first + index = gd.get_index(self._td.getsyspath(repo_uuid)) + index.set_to_tree(gd.resolve_ref_to_tree(gd.HEAD)) + self._to_push[gd] = (build_ref, index) + + rootinfo, = ((gd, index) for gd, (build_ref, index) + in self._to_push.iteritems() + if gd.get_config('morph.repository') == self._branch_root) + self._root, self._root_index = rootinfo + + def _register_cleanup(self, func, *args, **kwargs): + self._cleanup.append((func, args, kwargs)) + + def add_uncommitted_changes(self): + '''Add any uncommitted changes to temporary build GitIndexes''' + for gd, (build_ref, index) in self._to_push.iteritems(): + changed = [to_path for code, to_path, from_path + in index.get_uncommitted_changes()] + if not changed: + continue + yield gd, build_ref + index.add_files_from_working_tree(changed) + + @staticmethod + def _hash_morphologies(gd, morphologies, loader): + '''Hash morphologies and return object info''' + for morphology in morphologies: + loader.unset_defaults(morphology) + sha1 = gd.store_blob(loader.save_to_string(morphology)) + yield 0100644, sha1, morphology.filename + + def inject_build_refs(self, loader): + '''Update system and stratum morphologies to point to our branch. + + For all edited repositories, this alter the temporary GitIndex + of the morphs repositories to point their temporary build branch + versions. + + `loader` is a MorphologyLoader that is used to convert morphology + files into their in-memory representations and back again. + + ''' + root_repo = self._root.get_config('morph.repository') + root_ref = self._root.HEAD + morphs = morphlib.morphset.MorphologySet() + for morph in self._sb.load_all_morphologies(loader): + morphs.add_morphology(morph) + + sb_info = {} + for gd, (build_ref, index) in self._to_push.iteritems(): + repo, ref = gd.get_config('morph.repository'), gd.HEAD + sb_info[repo, ref] = (gd, build_ref) + + def filter(m, kind, spec): + return (spec['repo'], spec['ref']) in sb_info + def process(m, kind, spec): + repo, ref = spec['repo'], spec['ref'] + gd, build_ref = sb_info[repo, ref] + if (repo, ref) == (root_repo, root_ref): + spec['repo'] = None + spec['ref'] = None + return True + if not self._push_temporary: + spec['repo'] = urlparse.urljoin('file://', gd.dirname) + spec['ref'] = build_ref + return True + + morphs.traverse_specs(process, filter) + + if any(m.dirty for m in morphs.morphologies): + yield self._root + + self._root_index.add_files_from_index_info( + self._hash_morphologies(self._root, morphs.morphologies, loader)) + + def update_build_refs(self, name, email, uuid): + '''Commit changes in temporary GitIndexes to temporary branches. + + `name` and `email` are required to construct the commit author info. + `uuid` is used to identify each build uniquely and is included + in the commit message. + + A new commit is added to the temporary build branch of each of + the repositories in the SystemBranch with: + 1. The tree of anything currently in the temporary GitIndex. + This is the same as the current commit on HEAD unless + `add_uncommitted_changes` or `inject_build_refs` have + been called. + 2. the parent of the previous temporary commit, or the last + commit of the working tree if there has been no previous + commits + 3. author and committer email as specified by `email`, author + name of `name` and committer name of 'Morph (on behalf of + `name`)' + 4. commit message describing the current build using `uuid` + + ''' + commit_message = 'Morph build %s\n\nSystem branch: %s\n' % \ + (uuid, self._sb.system_branch_name) + author_name = name + committer_name = 'Morph (on behalf of %s)' % name + author_email = committer_email = email + + with morphlib.branchmanager.LocalRefManager() as lrm: + for gd, (build_ref, index) in self._to_push.iteritems(): + yield gd, build_ref + tree = index.write_tree() + try: + parent = gd.resolve_ref_to_commit(build_ref) + except morphlib.gitdir.InvalidRefError: + parent = gd.resolve_ref_to_commit(gd.HEAD) + + commit = gd.commit_tree(tree, parent=parent, + committer_name=committer_name, + committer_email=committer_email, + author_name=author_name, + author_email=author_email, + message=commit_message) + try: + old_commit = gd.resolve_ref_to_commit(build_ref) + except morphlib.gitdir.InvalidRefError: + lrm.add(gd, build_ref, commit) + else: + # NOTE: This will fail if build_ref pointed to a tag, + # due to resolve_ref_to_commit returning the + # commit id of tags, but since it's only morph + # that touches those refs, it should not be + # a problem. + lrm.update(gd, build_ref, commit, old_commit) + + def push_build_branches(self): + '''Push all temporary build branches to the remote repositories. + + This is a no-op if the BuildBranch was constructed with + `push_temporary` as False, so that the code flow for the user of + the BuildBranch can be the same when it can be pushed as when + it can't. + + ''' + # TODO: When BuildBranches become more context aware, if there + # are no uncommitted changes and the local versions are pushed + # we can skip pushing even if push_temporary is set. + # No uncommitted changes isn't sufficient reason to push the + # current HEAD + if self._push_temporary: + with morphlib.branchmanager.RemoteRefManager(False) as rrm: + for gd, build_ref in self._to_push.iterkeys(): + remote = gd.get_remote('origin') + yield gd, build_ref, remote + refspec = morphlib.gitdir.RefSpec(build_ref) + rrm.push(remote, refspec) + self._register_cleanup(rrm.close) + + @property + def root_repo_url(self): + '''URI of the repository that systems may be found in.''' + # TODO: When BuildBranches become more context aware, we only + # have to use the file:// URI when there's uncommitted changes + # and we can't push; or HEAD is not pushed and we can't push. + # All other times we can use the pushed branch + return (self._sb.get_config('branch.root') if self._push_temporary + else urlparse.urljoin('file://', self._root.dirname)) + + @property + def root_ref(self): + '''Name of the ref of the repository that systems may be found in.''' + # TODO: When BuildBranches become more context aware, this can be + # HEAD when there's no uncommitted changes and we're not pushing; + # or we are pushing and there's no uncommitted changes and HEAD + # has been pushed. + build_ref, index = self._to_push[self._root] + return build_ref + + def close(self): + '''Clean up any resources acquired during operation.''' + # TODO: This is a common pattern for our context managers, + # we could do with a way to share the common code. I suggest the + # ExitStack from python3.4 or the contextlib2 module. + exceptions = [] + while self._cleanup: + func, args, kwargs = self._cleanup.pop() + try: + func(*args, **kwargs) + except Exception, e: + exceptions.append(e) + if exceptions: + raise BuildBranchCleanupError(self, exceptions) diff --git a/without-test-modules b/without-test-modules index 4efcdb40..c34ba59c 100644 --- a/without-test-modules +++ b/without-test-modules @@ -28,3 +28,5 @@ morphlib/plugins/trovectl_plugin.py morphlib/plugins/gc_plugin.py morphlib/plugins/branch_and_merge_new_plugin.py morphlib/plugins/print_architecture_plugin.py +# Not unit tested, since it needs a full system branch +morphlib/buildbranch.py |