summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2013-11-15 15:09:40 +0000
committerRichard Maw <richard.maw@codethink.co.uk>2013-11-22 13:49:26 +0000
commit53d53ef939ee66de9b6dfbf5d2fe215fc7723400 (patch)
treeda35dfd0f45e427622ca2a51045b7d2a3122bda4
parent7e6bdb49957008b3981d570cbe8033627125efed (diff)
downloadmorph-53d53ef939ee66de9b6dfbf5d2fe215fc7723400.tar.gz
morphlib: Add branch context managers
This adds a LocalRefManager, which handles ref updates to local repositories (i.e. your workspace). It provides proxy methods for ref updates to a set of repositories. If an exception occurs in the body of the context manager, the updates will be rolled back to before the context manager was entered. The purpose for using a LocalRefManager instead of making the changes to the repositories directly, is to provide atomic updates to a set of refs in a set of repositories, where all refs are updated, or none are. This also adds a RemoteRefManager, which handles pushing branches to remote repositories. It provides a proxy push method, which will delete pushed branches, and re-push deleted branches after the context manager exits. Its purpose, instead of providing atomic updates to remote repositories, is to provide temporary branches. This is because it is used to provide temporary build branches. The difference between atomic update and temporary push, is that the remote branches are deleted when the context is left, rather than kept, as LocalRefManager does. The RemoteRefManager currently cannot provide the same atomicity guarantees as the LocalRefManager, so if there is a push between the branch being created and the RemoteRefManager cleaning it up, that change is lost without RemoteRefManager even knowing it existed. Git 1.8.5 will add functionality to make this possible.
-rw-r--r--morphlib/__init__.py1
-rw-r--r--morphlib/branchmanager.py158
-rw-r--r--morphlib/branchmanager_tests.py331
3 files changed, 490 insertions, 0 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 76b7a989..ef46866a 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -48,6 +48,7 @@ class Error(cliapp.AppException):
import artifact
import artifactcachereference
import artifactresolver
+import branchmanager
import bins
import buildcommand
import buildenvironment
diff --git a/morphlib/branchmanager.py b/morphlib/branchmanager.py
new file mode 100644
index 00000000..87a75ddc
--- /dev/null
+++ b/morphlib/branchmanager.py
@@ -0,0 +1,158 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import collections
+
+import morphlib
+
+
+class RefCleanupError(cliapp.AppException):
+ def __init__(self, primary_exception, exceptions):
+ self.exceptions = exceptions
+ self.ex_nr = ex_nr = len(exceptions)
+ self.primary_exception = primary_exception
+ cliapp.AppException.__init__(
+ self, '%(ex_nr)d exceptions caught when cleaning up '\
+ 'after exception: %(primary_exception)r: '\
+ '%(exceptions)r' % locals())
+
+
+class LocalRefManager(object):
+ '''Provide atomic update over a set of refs in a set of repositories.
+
+ Any ref changes made with the update, add and delete methods will
+ be reversed after the end of the with statement the LocalRefManager
+ is used in, if an exception is raised in the aforesaid with statement.
+
+ '''
+
+ def __init__(self):
+ self._cleanup = None
+
+ def __enter__(self):
+ self._cleanup = collections.deque()
+ return self
+
+ def __exit__(self, etype, evalue, estack):
+ # No exception was raised, so no cleanup is required
+ if (etype, evalue, estack) == (None, None, None):
+ return
+ exceptions = []
+ d = self._cleanup
+ while d:
+ op, args = d.pop()
+ try:
+ op(*args)
+ except Exception, e:
+ exceptions.append((op, args, e))
+ if exceptions:
+ raise RefCleanupError(evalue, exceptions)
+
+ def update(self, gd, ref, commit, old_commit, message=None):
+ '''Update a git repository's ref, reverting it on failure.
+
+ Use gd and the other parameters to update a ref to a new value,
+ and if an execption is raised in the body of the with statement
+ the LocalRefManager is used in, revert the update back to its
+ old value.
+
+ See morphlib.gitdir.update_ref for more information.
+
+ '''
+
+ gd.update_ref(ref, commit, old_commit, message)
+ # Register a cleanup callback of setting the ref back to its old value
+ self._cleanup.append((type(gd).update_ref,
+ (gd, ref, old_commit, commit,
+ message and 'Revert ' + message)))
+
+ def add(self, gd, ref, commit, message=None):
+ '''Add ref to a git repository, removing it on failure.
+
+ Use gd and the other parameters to add a new ref to the repository,
+ and if an execption is raised in the body of the with statement
+ the LocalRefManager is used in, delete the ref.
+
+ See morphlib.gitdir.add_ref for more information.
+
+ '''
+
+ gd.add_ref(ref, commit, message)
+ # Register a cleanup callback of deleting the newly added ref.
+ self._cleanup.append((type(gd).delete_ref, (gd, ref, commit,
+ message and 'Revert ' + message)))
+
+ def delete(self, gd, ref, old_commit, message=None):
+ '''Delete ref from a git repository, reinstating it on failure.
+
+ Use gd and the other parameters to delete an existing ref from
+ the repository, and if an execption is raised in the body of the
+ with statement the LocalRefManager is used in, re-create the ref.
+
+ See morphlib.gitdir.add_ref for more information.
+
+ '''
+
+ gd.delete_ref(ref, old_commit, message)
+ # Register a cleanup callback of replacing the deleted ref.
+ self._cleanup.append((type(gd).add_ref, (gd, ref, old_commit,
+ message and 'Revert ' + message)))
+
+
+class RemoteRefManager(object):
+ '''Provide temporary pushes to remote repositories.
+
+ Any ref changes made with the push method will be reversed after
+ the end of the with statement the RemoteRefManager is used in.
+
+ '''
+
+ def __init__(self):
+ self._cleanup = None
+
+ def __enter__(self):
+ self._cleanup = collections.deque()
+ return self
+
+ def __exit__(self, etype, evalue, estack):
+ exceptions = []
+ d = self._cleanup
+ while d:
+ remote, refspecs = d.pop()
+ try:
+ remote.push(*refspecs)
+ except Exception, e:
+ exceptions.append((remote, refspecs, e))
+ if exceptions:
+ raise RefCleanupError(evalue, exceptions)
+
+ def push(self, remote, *refspecs):
+ '''Push refspecs to remote and revert on failure.
+
+ Push the specified refspecs to the remote and reverse the change
+ after the end of the block the with statement the RemoteRefManager
+ is used in.
+
+ '''
+
+ # Calculate the refspecs required to undo the pushed changes.
+ delete_specs = tuple(rs.revert() for rs in refspecs)
+ result = remote.push(*refspecs)
+ # Register cleanup after pushing, so that if this push fails,
+ # we don't try to undo it.
+ self._cleanup.append((remote, delete_specs))
+ return result
diff --git a/morphlib/branchmanager_tests.py b/morphlib/branchmanager_tests.py
new file mode 100644
index 00000000..9bba7f2e
--- /dev/null
+++ b/morphlib/branchmanager_tests.py
@@ -0,0 +1,331 @@
+# Copyright (C) 2013 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import os
+import shutil
+import tempfile
+import unittest
+
+import morphlib
+
+
+class LocalRefManagerTests(unittest.TestCase):
+
+ REPO_COUNT = 3
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp()
+ self.repos = []
+ for i in xrange(self.REPO_COUNT):
+ dirname = os.path.join(self.tempdir, 'repo%d' % i)
+ os.mkdir(dirname)
+ gd = morphlib.gitdir.init(dirname)
+ with open(os.path.join(dirname, 'foo'), 'w') as f:
+ f.write('dummy text\n')
+ gd._runcmd(['git', 'add', '.'])
+ gd._runcmd(['git', 'commit', '-m', 'Initial commit'])
+ gd._runcmd(['git', 'checkout', '-b', 'dev-branch'])
+ with open(os.path.join(dirname, 'foo'), 'w') as f:
+ f.write('updated text\n')
+ gd._runcmd(['git', 'add', '.'])
+ gd._runcmd(['git', 'commit', '-m', 'Second commit'])
+ self.repos.append(gd)
+
+ def tearDown(self):
+ shutil.rmtree(self.tempdir)
+
+ @staticmethod
+ def lrm(*args, **kwargs):
+ return morphlib.branchmanager.LocalRefManager(*args, **kwargs)
+
+ def test_refs_added(self):
+ refinfo = []
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ commit = gd.resolve_ref_to_commit('refs/heads/master')
+ refinfo.append(commit)
+ lrm.add(gd, 'refs/heads/create%d' % i, commit)
+ for i, gd in enumerate(self.repos):
+ self.assertEqual(gd.resolve_ref_to_commit(
+ 'refs/heads/create%d' % i),
+ refinfo[i])
+
+ def test_add_rollback(self):
+ with self.assertRaises(Exception):
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ commit = gd.resolve_ref_to_commit('refs/heads/master')
+ lrm.add(gd, 'refs/heads/create%d' % i, commit)
+ raise Exception()
+ for i, gd in enumerate(self.repos):
+ with self.assertRaises(morphlib.gitdir.InvalidRefError):
+ gd.resolve_ref_to_commit('refs/heads/create%d' % i)
+
+ def test_add_rollback_failure(self):
+ failure_exception = Exception()
+ with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm:
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ ref = 'refs/heads/create%d' % i
+ commit = gd.resolve_ref_to_commit('refs/heads/master')
+ lrm.add(gd, ref, commit)
+ # Make changes independent of LRM, so that rollback fails
+ new_commit = gd.resolve_ref_to_commit(
+ 'refs/heads/dev-branch')
+ gd.update_ref(ref, new_commit, commit)
+ raise failure_exception
+ self.assertEqual(cm.exception.primary_exception, failure_exception)
+ self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions],
+ [morphlib.gitdir.RefDeleteError] * self.REPO_COUNT)
+
+ def test_refs_updated(self):
+ refinfo = []
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ old_master = gd.resolve_ref_to_commit('refs/heads/master')
+ commit = gd.resolve_ref_to_commit('refs/heads/dev-branch')
+ refinfo.append(commit)
+ lrm.update(gd, 'refs/heads/master', commit, old_master)
+ for i, gd in enumerate(self.repos):
+ self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'),
+ refinfo[i])
+
+ def test_update_rollback(self):
+ refinfo = []
+ with self.assertRaises(Exception):
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ old_master = gd.resolve_ref_to_commit('refs/heads/master')
+ commit = gd.resolve_ref_to_commit('refs/heads/dev-branch')
+ refinfo.append(old_master)
+ lrm.update(gd, 'refs/heads/master', commit, old_master)
+ raise Exception()
+ for i, gd in enumerate(self.repos):
+ self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'),
+ refinfo[i])
+
+ def test_update_rollback_failure(self):
+ failure_exception = Exception()
+ with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm:
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ old_master = gd.resolve_ref_to_commit('refs/heads/master')
+ commit = gd.resolve_ref_to_commit('refs/heads/dev-branch')
+ lrm.update(gd, 'refs/heads/master', commit, old_master)
+ # Delete the ref, so rollback fails
+ gd.delete_ref('refs/heads/master', commit)
+ raise failure_exception
+ self.assertEqual(cm.exception.primary_exception, failure_exception)
+ self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions],
+ [morphlib.gitdir.RefUpdateError] * self.REPO_COUNT)
+
+ def test_refs_deleted(self):
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ commit = gd.resolve_ref_to_commit('refs/heads/master')
+ lrm.delete(gd, 'refs/heads/master', commit)
+ for i, gd in enumerate(self.repos):
+ self.assertRaises(morphlib.gitdir.InvalidRefError,
+ gd.resolve_ref_to_commit, 'refs/heads/master')
+
+ def test_delete_rollback(self):
+ refinfo = []
+ with self.assertRaises(Exception):
+ with self.lrm() as lrm:
+ for i, gd in enumerate(self.repos):
+ commit = gd.resolve_ref_to_commit('refs/heads/master')
+ refinfo.append(commit)
+ lrm.delete(gd, 'refs/heads/master', commit)
+ raise Exception()
+ for i, gd in enumerate(self.repos):
+ self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'),
+ refinfo[i])
+
+ def test_delete_rollback_failure(self):
+ failure_exception = Exception()
+ with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm:
+ with self.lrm() as lrm:
+ for gd in self.repos:
+ commit = gd.resolve_ref_to_commit('refs/heads/master')
+ lrm.delete(gd, 'refs/heads/master', commit)
+ gd.add_ref('refs/heads/master', commit)
+ raise failure_exception
+ self.assertEqual(cm.exception.primary_exception, failure_exception)
+ self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions],
+ [morphlib.gitdir.RefAddError] * self.REPO_COUNT)
+
+
+class RemoteRefManagerTests(unittest.TestCase):
+
+ TARGET_COUNT = 2
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp()
+ self.source = os.path.join(self.tempdir, 'source')
+ os.mkdir(self.source)
+ self.sgd = morphlib.gitdir.init(self.source)
+ with open(os.path.join(self.source, 'foo'), 'w') as f:
+ f.write('dummy text\n')
+ self.sgd._runcmd(['git', 'add', '.'])
+ self.sgd._runcmd(['git', 'commit', '-m', 'Initial commit'])
+ self.sgd._runcmd(['git', 'checkout', '-b', 'dev-branch'])
+ with open(os.path.join(self.source, 'foo'), 'w') as f:
+ f.write('updated text\n')
+ self.sgd._runcmd(['git', 'add', '.'])
+ self.sgd._runcmd(['git', 'commit', '-m', 'Second commit'])
+ self.sgd._runcmd(['git', 'checkout', '--orphan', 'no-ff'])
+ with open(os.path.join(self.source, 'foo'), 'w') as f:
+ f.write('parallel dimension text\n')
+ self.sgd._runcmd(['git', 'add', '.'])
+ self.sgd._runcmd(['git', 'commit', '-m', 'Non-fast-forward commit'])
+
+ self.remotes = []
+ for i in xrange(self.TARGET_COUNT):
+ name = 'remote-%d' % i
+ dirname = os.path.join(self.tempdir, name)
+
+ # Allow deleting HEAD
+ cliapp.runcmd(['git', 'init', '--bare', dirname])
+ gd = morphlib.gitdir.GitDirectory(dirname)
+ gd.set_config('receive.denyDeleteCurrent', 'warn')
+
+ self.sgd._runcmd(['git', 'remote', 'add', name, dirname])
+ self.remotes.append((name, dirname, gd))
+
+ def tearDown(self):
+ shutil.rmtree(self.tempdir)
+
+ @staticmethod
+ def list_refs(gd):
+ out = gd._runcmd(['git', 'for-each-ref',
+ '--format=%(refname)%00%(objectname)%00'])
+ return dict(line.split('\0') for line in
+ out.strip('\0\n').split('\0\n') if line)
+
+ def push_creates(self, rrm):
+ for name, dirname, gd in self.remotes:
+ rrm.push(self.sgd.get_remote(name),
+ morphlib.gitdir.RefSpec('refs/heads/master'),
+ morphlib.gitdir.RefSpec('refs/heads/dev-branch'))
+
+ def push_deletes(self, rrm):
+ null_commit = '0' * 40
+ master_commit = self.sgd.resolve_ref_to_commit('refs/heads/master')
+ dev_commit = self.sgd.resolve_ref_to_commit('refs/heads/dev-branch')
+ for name, dirname, gd in self.remotes:
+ rrm.push(self.sgd.get_remote(name),
+ morphlib.gitdir.RefSpec(
+ source=null_commit,
+ target='refs/heads/master',
+ require=master_commit),
+ morphlib.gitdir.RefSpec(
+ source=null_commit,
+ target='refs/heads/dev-branch',
+ require=dev_commit))
+
+ def assert_no_remote_branches(self):
+ for name, dirname, gd in self.remotes:
+ self.assertEqual(self.list_refs(gd), {})
+
+ def assert_remote_branches(self):
+ for name, dirname, gd in self.remotes:
+ for name, sha1 in self.list_refs(gd).iteritems():
+ self.assertEqual(self.sgd.resolve_ref_to_commit(name), sha1)
+
+ def test_rollback_after_create_success(self):
+ with morphlib.branchmanager.RemoteRefManager() as rrm:
+ self.push_creates(rrm)
+ self.assert_remote_branches()
+ self.assert_no_remote_branches()
+
+ def test_rollback_after_create_failure(self):
+ failure_exception = Exception()
+ with self.assertRaises(Exception) as cm:
+ with morphlib.branchmanager.RemoteRefManager() as rrm:
+ self.push_creates(rrm)
+ raise failure_exception
+ self.assertEqual(cm.exception, failure_exception)
+ self.assert_no_remote_branches()
+
+ @unittest.skip('No way to have conditional delete until Git 1.8.5')
+ def test_rollback_after_create_cleanup_failure(self):
+ failure_exception = Exception()
+ with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm:
+ with morphlib.branchmanager.RemoteRefManager() as rrm:
+ self.push_creates(rrm)
+
+ # Break rollback with a new non-ff commit on master
+ no_ff = self.sgd.resolve_ref_to_commit('no-ff')
+ master = 'refs/heads/master'
+ master_commit = \
+ self.sgd.resolve_ref_to_commit('refs/heads/master')
+ for name, dirname, gd in self.remotes:
+ r = self.sgd.get_remote(name)
+ r.push(morphlib.gitdir.RefSpec(source=no_ff, target=master,
+ require=master_commit,
+ force=True))
+
+ raise failure_exception
+ self.assertEqual(cm.exception.primary_exception, failure_exception)
+ self.assert_no_remote_branches()
+
+ def test_rollback_after_deletes_success(self):
+ for name, dirname, gd in self.remotes:
+ self.sgd.get_remote(name).push(
+ morphlib.gitdir.RefSpec('master'),
+ morphlib.gitdir.RefSpec('dev-branch'))
+ self.assert_remote_branches()
+ with morphlib.branchmanager.RemoteRefManager() as rrm:
+ self.push_deletes(rrm)
+ self.assert_no_remote_branches()
+ self.assert_remote_branches()
+
+ def test_rollback_after_deletes_failure(self):
+ failure_exception = Exception()
+ for name, dirname, gd in self.remotes:
+ self.sgd.get_remote(name).push(
+ morphlib.gitdir.RefSpec('master'),
+ morphlib.gitdir.RefSpec('dev-branch'))
+ self.assert_remote_branches()
+ with self.assertRaises(Exception) as cm:
+ with morphlib.branchmanager.RemoteRefManager() as rrm:
+ self.push_deletes(rrm)
+ raise failure_exception
+ self.assertEqual(cm.exception, failure_exception)
+ self.assert_remote_branches()
+
+ def test_rollback_after_deletes_cleanup_failure(self):
+ failure_exception = Exception()
+ for name, dirname, gd in self.remotes:
+ self.sgd.get_remote(name).push(
+ morphlib.gitdir.RefSpec('master'),
+ morphlib.gitdir.RefSpec('dev-branch'))
+ with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm:
+ with morphlib.branchmanager.RemoteRefManager() as rrm:
+ self.push_deletes(rrm)
+
+ # Break rollback with a new non-ff commit on master
+ no_ff = self.sgd.resolve_ref_to_commit('no-ff')
+ master = 'refs/heads/master'
+ master_commit = \
+ self.sgd.resolve_ref_to_commit('refs/heads/master')
+ for name, dirname, gd in self.remotes:
+ r = self.sgd.get_remote(name)
+ r.push(morphlib.gitdir.RefSpec(source=no_ff, target=master,
+ require=master_commit))
+
+ raise failure_exception
+ self.assertEqual(cm.exception.primary_exception, failure_exception)
+