diff options
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/branchmanager.py | 158 | ||||
| -rw-r--r-- | morphlib/branchmanager_tests.py | 331 |
3 files changed, 490 insertions, 0 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 76b7a989..ef46866a 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -48,6 +48,7 @@ class Error(cliapp.AppException): import artifact import artifactcachereference import artifactresolver +import branchmanager import bins import buildcommand import buildenvironment diff --git a/morphlib/branchmanager.py b/morphlib/branchmanager.py new file mode 100644 index 00000000..87a75ddc --- /dev/null +++ b/morphlib/branchmanager.py @@ -0,0 +1,158 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import collections + +import morphlib + + +class RefCleanupError(cliapp.AppException): + def __init__(self, primary_exception, exceptions): + self.exceptions = exceptions + self.ex_nr = ex_nr = len(exceptions) + self.primary_exception = primary_exception + cliapp.AppException.__init__( + self, '%(ex_nr)d exceptions caught when cleaning up '\ + 'after exception: %(primary_exception)r: '\ + '%(exceptions)r' % locals()) + + +class LocalRefManager(object): + '''Provide atomic update over a set of refs in a set of repositories. + + Any ref changes made with the update, add and delete methods will + be reversed after the end of the with statement the LocalRefManager + is used in, if an exception is raised in the aforesaid with statement. + + ''' + + def __init__(self): + self._cleanup = None + + def __enter__(self): + self._cleanup = collections.deque() + return self + + def __exit__(self, etype, evalue, estack): + # No exception was raised, so no cleanup is required + if (etype, evalue, estack) == (None, None, None): + return + exceptions = [] + d = self._cleanup + while d: + op, args = d.pop() + try: + op(*args) + except Exception, e: + exceptions.append((op, args, e)) + if exceptions: + raise RefCleanupError(evalue, exceptions) + + def update(self, gd, ref, commit, old_commit, message=None): + '''Update a git repository's ref, reverting it on failure. + + Use gd and the other parameters to update a ref to a new value, + and if an execption is raised in the body of the with statement + the LocalRefManager is used in, revert the update back to its + old value. + + See morphlib.gitdir.update_ref for more information. + + ''' + + gd.update_ref(ref, commit, old_commit, message) + # Register a cleanup callback of setting the ref back to its old value + self._cleanup.append((type(gd).update_ref, + (gd, ref, old_commit, commit, + message and 'Revert ' + message))) + + def add(self, gd, ref, commit, message=None): + '''Add ref to a git repository, removing it on failure. + + Use gd and the other parameters to add a new ref to the repository, + and if an execption is raised in the body of the with statement + the LocalRefManager is used in, delete the ref. + + See morphlib.gitdir.add_ref for more information. + + ''' + + gd.add_ref(ref, commit, message) + # Register a cleanup callback of deleting the newly added ref. + self._cleanup.append((type(gd).delete_ref, (gd, ref, commit, + message and 'Revert ' + message))) + + def delete(self, gd, ref, old_commit, message=None): + '''Delete ref from a git repository, reinstating it on failure. + + Use gd and the other parameters to delete an existing ref from + the repository, and if an execption is raised in the body of the + with statement the LocalRefManager is used in, re-create the ref. + + See morphlib.gitdir.add_ref for more information. + + ''' + + gd.delete_ref(ref, old_commit, message) + # Register a cleanup callback of replacing the deleted ref. + self._cleanup.append((type(gd).add_ref, (gd, ref, old_commit, + message and 'Revert ' + message))) + + +class RemoteRefManager(object): + '''Provide temporary pushes to remote repositories. + + Any ref changes made with the push method will be reversed after + the end of the with statement the RemoteRefManager is used in. + + ''' + + def __init__(self): + self._cleanup = None + + def __enter__(self): + self._cleanup = collections.deque() + return self + + def __exit__(self, etype, evalue, estack): + exceptions = [] + d = self._cleanup + while d: + remote, refspecs = d.pop() + try: + remote.push(*refspecs) + except Exception, e: + exceptions.append((remote, refspecs, e)) + if exceptions: + raise RefCleanupError(evalue, exceptions) + + def push(self, remote, *refspecs): + '''Push refspecs to remote and revert on failure. + + Push the specified refspecs to the remote and reverse the change + after the end of the block the with statement the RemoteRefManager + is used in. + + ''' + + # Calculate the refspecs required to undo the pushed changes. + delete_specs = tuple(rs.revert() for rs in refspecs) + result = remote.push(*refspecs) + # Register cleanup after pushing, so that if this push fails, + # we don't try to undo it. + self._cleanup.append((remote, delete_specs)) + return result diff --git a/morphlib/branchmanager_tests.py b/morphlib/branchmanager_tests.py new file mode 100644 index 00000000..9bba7f2e --- /dev/null +++ b/morphlib/branchmanager_tests.py @@ -0,0 +1,331 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class LocalRefManagerTests(unittest.TestCase): + + REPO_COUNT = 3 + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.repos = [] + for i in xrange(self.REPO_COUNT): + dirname = os.path.join(self.tempdir, 'repo%d' % i) + os.mkdir(dirname) + gd = morphlib.gitdir.init(dirname) + with open(os.path.join(dirname, 'foo'), 'w') as f: + f.write('dummy text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + gd._runcmd(['git', 'checkout', '-b', 'dev-branch']) + with open(os.path.join(dirname, 'foo'), 'w') as f: + f.write('updated text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Second commit']) + self.repos.append(gd) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + @staticmethod + def lrm(*args, **kwargs): + return morphlib.branchmanager.LocalRefManager(*args, **kwargs) + + def test_refs_added(self): + refinfo = [] + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + refinfo.append(commit) + lrm.add(gd, 'refs/heads/create%d' % i, commit) + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit( + 'refs/heads/create%d' % i), + refinfo[i]) + + def test_add_rollback(self): + with self.assertRaises(Exception): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.add(gd, 'refs/heads/create%d' % i, commit) + raise Exception() + for i, gd in enumerate(self.repos): + with self.assertRaises(morphlib.gitdir.InvalidRefError): + gd.resolve_ref_to_commit('refs/heads/create%d' % i) + + def test_add_rollback_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + ref = 'refs/heads/create%d' % i + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.add(gd, ref, commit) + # Make changes independent of LRM, so that rollback fails + new_commit = gd.resolve_ref_to_commit( + 'refs/heads/dev-branch') + gd.update_ref(ref, new_commit, commit) + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions], + [morphlib.gitdir.RefDeleteError] * self.REPO_COUNT) + + def test_refs_updated(self): + refinfo = [] + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + old_master = gd.resolve_ref_to_commit('refs/heads/master') + commit = gd.resolve_ref_to_commit('refs/heads/dev-branch') + refinfo.append(commit) + lrm.update(gd, 'refs/heads/master', commit, old_master) + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'), + refinfo[i]) + + def test_update_rollback(self): + refinfo = [] + with self.assertRaises(Exception): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + old_master = gd.resolve_ref_to_commit('refs/heads/master') + commit = gd.resolve_ref_to_commit('refs/heads/dev-branch') + refinfo.append(old_master) + lrm.update(gd, 'refs/heads/master', commit, old_master) + raise Exception() + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'), + refinfo[i]) + + def test_update_rollback_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + old_master = gd.resolve_ref_to_commit('refs/heads/master') + commit = gd.resolve_ref_to_commit('refs/heads/dev-branch') + lrm.update(gd, 'refs/heads/master', commit, old_master) + # Delete the ref, so rollback fails + gd.delete_ref('refs/heads/master', commit) + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions], + [morphlib.gitdir.RefUpdateError] * self.REPO_COUNT) + + def test_refs_deleted(self): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.delete(gd, 'refs/heads/master', commit) + for i, gd in enumerate(self.repos): + self.assertRaises(morphlib.gitdir.InvalidRefError, + gd.resolve_ref_to_commit, 'refs/heads/master') + + def test_delete_rollback(self): + refinfo = [] + with self.assertRaises(Exception): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + refinfo.append(commit) + lrm.delete(gd, 'refs/heads/master', commit) + raise Exception() + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'), + refinfo[i]) + + def test_delete_rollback_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with self.lrm() as lrm: + for gd in self.repos: + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.delete(gd, 'refs/heads/master', commit) + gd.add_ref('refs/heads/master', commit) + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions], + [morphlib.gitdir.RefAddError] * self.REPO_COUNT) + + +class RemoteRefManagerTests(unittest.TestCase): + + TARGET_COUNT = 2 + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.source = os.path.join(self.tempdir, 'source') + os.mkdir(self.source) + self.sgd = morphlib.gitdir.init(self.source) + with open(os.path.join(self.source, 'foo'), 'w') as f: + f.write('dummy text\n') + self.sgd._runcmd(['git', 'add', '.']) + self.sgd._runcmd(['git', 'commit', '-m', 'Initial commit']) + self.sgd._runcmd(['git', 'checkout', '-b', 'dev-branch']) + with open(os.path.join(self.source, 'foo'), 'w') as f: + f.write('updated text\n') + self.sgd._runcmd(['git', 'add', '.']) + self.sgd._runcmd(['git', 'commit', '-m', 'Second commit']) + self.sgd._runcmd(['git', 'checkout', '--orphan', 'no-ff']) + with open(os.path.join(self.source, 'foo'), 'w') as f: + f.write('parallel dimension text\n') + self.sgd._runcmd(['git', 'add', '.']) + self.sgd._runcmd(['git', 'commit', '-m', 'Non-fast-forward commit']) + + self.remotes = [] + for i in xrange(self.TARGET_COUNT): + name = 'remote-%d' % i + dirname = os.path.join(self.tempdir, name) + + # Allow deleting HEAD + cliapp.runcmd(['git', 'init', '--bare', dirname]) + gd = morphlib.gitdir.GitDirectory(dirname) + gd.set_config('receive.denyDeleteCurrent', 'warn') + + self.sgd._runcmd(['git', 'remote', 'add', name, dirname]) + self.remotes.append((name, dirname, gd)) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + @staticmethod + def list_refs(gd): + out = gd._runcmd(['git', 'for-each-ref', + '--format=%(refname)%00%(objectname)%00']) + return dict(line.split('\0') for line in + out.strip('\0\n').split('\0\n') if line) + + def push_creates(self, rrm): + for name, dirname, gd in self.remotes: + rrm.push(self.sgd.get_remote(name), + morphlib.gitdir.RefSpec('refs/heads/master'), + morphlib.gitdir.RefSpec('refs/heads/dev-branch')) + + def push_deletes(self, rrm): + null_commit = '0' * 40 + master_commit = self.sgd.resolve_ref_to_commit('refs/heads/master') + dev_commit = self.sgd.resolve_ref_to_commit('refs/heads/dev-branch') + for name, dirname, gd in self.remotes: + rrm.push(self.sgd.get_remote(name), + morphlib.gitdir.RefSpec( + source=null_commit, + target='refs/heads/master', + require=master_commit), + morphlib.gitdir.RefSpec( + source=null_commit, + target='refs/heads/dev-branch', + require=dev_commit)) + + def assert_no_remote_branches(self): + for name, dirname, gd in self.remotes: + self.assertEqual(self.list_refs(gd), {}) + + def assert_remote_branches(self): + for name, dirname, gd in self.remotes: + for name, sha1 in self.list_refs(gd).iteritems(): + self.assertEqual(self.sgd.resolve_ref_to_commit(name), sha1) + + def test_rollback_after_create_success(self): + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_creates(rrm) + self.assert_remote_branches() + self.assert_no_remote_branches() + + def test_rollback_after_create_failure(self): + failure_exception = Exception() + with self.assertRaises(Exception) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_creates(rrm) + raise failure_exception + self.assertEqual(cm.exception, failure_exception) + self.assert_no_remote_branches() + + @unittest.skip('No way to have conditional delete until Git 1.8.5') + def test_rollback_after_create_cleanup_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_creates(rrm) + + # Break rollback with a new non-ff commit on master + no_ff = self.sgd.resolve_ref_to_commit('no-ff') + master = 'refs/heads/master' + master_commit = \ + self.sgd.resolve_ref_to_commit('refs/heads/master') + for name, dirname, gd in self.remotes: + r = self.sgd.get_remote(name) + r.push(morphlib.gitdir.RefSpec(source=no_ff, target=master, + require=master_commit, + force=True)) + + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assert_no_remote_branches() + + def test_rollback_after_deletes_success(self): + for name, dirname, gd in self.remotes: + self.sgd.get_remote(name).push( + morphlib.gitdir.RefSpec('master'), + morphlib.gitdir.RefSpec('dev-branch')) + self.assert_remote_branches() + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_deletes(rrm) + self.assert_no_remote_branches() + self.assert_remote_branches() + + def test_rollback_after_deletes_failure(self): + failure_exception = Exception() + for name, dirname, gd in self.remotes: + self.sgd.get_remote(name).push( + morphlib.gitdir.RefSpec('master'), + morphlib.gitdir.RefSpec('dev-branch')) + self.assert_remote_branches() + with self.assertRaises(Exception) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_deletes(rrm) + raise failure_exception + self.assertEqual(cm.exception, failure_exception) + self.assert_remote_branches() + + def test_rollback_after_deletes_cleanup_failure(self): + failure_exception = Exception() + for name, dirname, gd in self.remotes: + self.sgd.get_remote(name).push( + morphlib.gitdir.RefSpec('master'), + morphlib.gitdir.RefSpec('dev-branch')) + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_deletes(rrm) + + # Break rollback with a new non-ff commit on master + no_ff = self.sgd.resolve_ref_to_commit('no-ff') + master = 'refs/heads/master' + master_commit = \ + self.sgd.resolve_ref_to_commit('refs/heads/master') + for name, dirname, gd in self.remotes: + r = self.sgd.get_remote(name) + r.push(morphlib.gitdir.RefSpec(source=no_ff, target=master, + require=master_commit)) + + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + |