diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2013-11-22 15:06:13 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2013-11-22 15:06:13 +0000 |
commit | b2b618f71d63290efbc849650459ce6cd467621d (patch) | |
tree | da35dfd0f45e427622ca2a51045b7d2a3122bda4 | |
parent | 0b3ec68ce46f638e79e52f9f97f26727d9c4daa1 (diff) | |
parent | 53d53ef939ee66de9b6dfbf5d2fe215fc7723400 (diff) | |
download | morph-b2b618f71d63290efbc849650459ce6cd467621d.tar.gz |
Merge remote-tracking branch 'origin/baserock/richardmaw/S9475/build-refactor-foundations-v2'
-rwxr-xr-x | check | 81 | ||||
-rw-r--r-- | morphlib/__init__.py | 2 | ||||
-rw-r--r-- | morphlib/branchmanager.py | 158 | ||||
-rw-r--r-- | morphlib/branchmanager_tests.py | 331 | ||||
-rw-r--r-- | morphlib/gitdir.py | 484 | ||||
-rw-r--r-- | morphlib/gitdir_tests.py | 315 | ||||
-rw-r--r-- | morphlib/gitindex.py | 159 | ||||
-rw-r--r-- | morphlib/gitindex_tests.py | 92 | ||||
-rw-r--r-- | morphlib/morphset.py | 12 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 18 | ||||
-rw-r--r-- | morphlib/sysbranchdir.py | 3 | ||||
-rw-r--r-- | morphlib/util.py | 10 | ||||
-rw-r--r-- | morphlib/util_tests.py | 11 | ||||
-rwxr-xr-x | scripts/check-silliness | 63 | ||||
-rw-r--r-- | yarns/morph.shell-lib | 3 |
15 files changed, 1599 insertions, 143 deletions
@@ -42,6 +42,28 @@ case "$PYTHONPATH" in esac export PYTHONPATH +# Run the style checks + +errors=0 +if [ -d .git ]; +then + echo "Checking copyright statements" + if ! (git ls-files -z | xargs -0r scripts/check-copyright-year); then + errors=1 + fi + + echo 'Checking source code for silliness' + if ! (git ls-files | + grep -v '\.gz$' | + grep -Ev 'tests[^/]*/.*\.std(out|err)' | + grep -vF 'tests.build/build-system-autotools.script' | + xargs -r scripts/check-silliness); then + errors=1 + fi +fi +if [ "$errors" != 0 ]; then + exit "$errors" +fi # Clean up artifacts from previous (possibly failed) runs, build, # and run the tests. @@ -97,62 +119,3 @@ then else echo "NOT RUNNING tests.as-root (requires PyYAML)" fi - -if [ -d .git ]; -then - echo "Checking copyright statements" - git ls-files | xargs scripts/check-copyright-year - - echo 'Checking source code for silliness' - git ls-files | - grep -v '\.gz$' | - grep -Ev 'tests[^/]*/.*\.std(out|err)' | - grep -vF 'tests.build/build-system-autotools.script' | - while read x - do - if tr -cd '\t' < "$x" | grep . > /dev/null - then - echo "ERROR: $x contains TAB characters" 1>&2 - grep -n -F "$(printf "\t")" "$x" 1>&2 - errors=1 - fi - - case "$x" in - # Excluding yarn files since it's not possible to split up the - # IMPLEMENTS lines of them - *.yarn) ;; - *) - if awk 'length > 79' "$x" | grep . > /dev/null - then - echo "ERROR: $x has lines longer than 79 chars" 1>&2 - awk 'length > 79 { print NR, $0 }' "$x" 1>&2 - errors=1 - fi - ;; - esac - - case "$x" in - *.py) - if head -1 "$x" | grep '^#!' > /dev/null - then - echo "ERROR: $x has a hashbang" 1>&2 - errors=1 - fi - if grep except: "$x" - then - echo "ERROR: $x has a bare except:" 1>&2 - errors=1 - fi - ;; - esac - done - - echo 'Checking for executable *.py files' - find . -type f -name '*.py' -perm +111 | - while read x - do - echo "ERROR: $x is executable" 1>&2 - errors=1 - done -fi -exit $errors diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 4954f812..ef46866a 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -48,6 +48,7 @@ class Error(cliapp.AppException): import artifact import artifactcachereference import artifactresolver +import branchmanager import bins import buildcommand import buildenvironment @@ -60,6 +61,7 @@ import extractedtarball import fsutils import git import gitdir +import gitindex import localartifactcache import localrepocache import mountableimage diff --git a/morphlib/branchmanager.py b/morphlib/branchmanager.py new file mode 100644 index 00000000..87a75ddc --- /dev/null +++ b/morphlib/branchmanager.py @@ -0,0 +1,158 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import collections + +import morphlib + + +class RefCleanupError(cliapp.AppException): + def __init__(self, primary_exception, exceptions): + self.exceptions = exceptions + self.ex_nr = ex_nr = len(exceptions) + self.primary_exception = primary_exception + cliapp.AppException.__init__( + self, '%(ex_nr)d exceptions caught when cleaning up '\ + 'after exception: %(primary_exception)r: '\ + '%(exceptions)r' % locals()) + + +class LocalRefManager(object): + '''Provide atomic update over a set of refs in a set of repositories. + + Any ref changes made with the update, add and delete methods will + be reversed after the end of the with statement the LocalRefManager + is used in, if an exception is raised in the aforesaid with statement. + + ''' + + def __init__(self): + self._cleanup = None + + def __enter__(self): + self._cleanup = collections.deque() + return self + + def __exit__(self, etype, evalue, estack): + # No exception was raised, so no cleanup is required + if (etype, evalue, estack) == (None, None, None): + return + exceptions = [] + d = self._cleanup + while d: + op, args = d.pop() + try: + op(*args) + except Exception, e: + exceptions.append((op, args, e)) + if exceptions: + raise RefCleanupError(evalue, exceptions) + + def update(self, gd, ref, commit, old_commit, message=None): + '''Update a git repository's ref, reverting it on failure. + + Use gd and the other parameters to update a ref to a new value, + and if an execption is raised in the body of the with statement + the LocalRefManager is used in, revert the update back to its + old value. + + See morphlib.gitdir.update_ref for more information. + + ''' + + gd.update_ref(ref, commit, old_commit, message) + # Register a cleanup callback of setting the ref back to its old value + self._cleanup.append((type(gd).update_ref, + (gd, ref, old_commit, commit, + message and 'Revert ' + message))) + + def add(self, gd, ref, commit, message=None): + '''Add ref to a git repository, removing it on failure. + + Use gd and the other parameters to add a new ref to the repository, + and if an execption is raised in the body of the with statement + the LocalRefManager is used in, delete the ref. + + See morphlib.gitdir.add_ref for more information. + + ''' + + gd.add_ref(ref, commit, message) + # Register a cleanup callback of deleting the newly added ref. + self._cleanup.append((type(gd).delete_ref, (gd, ref, commit, + message and 'Revert ' + message))) + + def delete(self, gd, ref, old_commit, message=None): + '''Delete ref from a git repository, reinstating it on failure. + + Use gd and the other parameters to delete an existing ref from + the repository, and if an execption is raised in the body of the + with statement the LocalRefManager is used in, re-create the ref. + + See morphlib.gitdir.add_ref for more information. + + ''' + + gd.delete_ref(ref, old_commit, message) + # Register a cleanup callback of replacing the deleted ref. + self._cleanup.append((type(gd).add_ref, (gd, ref, old_commit, + message and 'Revert ' + message))) + + +class RemoteRefManager(object): + '''Provide temporary pushes to remote repositories. + + Any ref changes made with the push method will be reversed after + the end of the with statement the RemoteRefManager is used in. + + ''' + + def __init__(self): + self._cleanup = None + + def __enter__(self): + self._cleanup = collections.deque() + return self + + def __exit__(self, etype, evalue, estack): + exceptions = [] + d = self._cleanup + while d: + remote, refspecs = d.pop() + try: + remote.push(*refspecs) + except Exception, e: + exceptions.append((remote, refspecs, e)) + if exceptions: + raise RefCleanupError(evalue, exceptions) + + def push(self, remote, *refspecs): + '''Push refspecs to remote and revert on failure. + + Push the specified refspecs to the remote and reverse the change + after the end of the block the with statement the RemoteRefManager + is used in. + + ''' + + # Calculate the refspecs required to undo the pushed changes. + delete_specs = tuple(rs.revert() for rs in refspecs) + result = remote.push(*refspecs) + # Register cleanup after pushing, so that if this push fails, + # we don't try to undo it. + self._cleanup.append((remote, delete_specs)) + return result diff --git a/morphlib/branchmanager_tests.py b/morphlib/branchmanager_tests.py new file mode 100644 index 00000000..9bba7f2e --- /dev/null +++ b/morphlib/branchmanager_tests.py @@ -0,0 +1,331 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class LocalRefManagerTests(unittest.TestCase): + + REPO_COUNT = 3 + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.repos = [] + for i in xrange(self.REPO_COUNT): + dirname = os.path.join(self.tempdir, 'repo%d' % i) + os.mkdir(dirname) + gd = morphlib.gitdir.init(dirname) + with open(os.path.join(dirname, 'foo'), 'w') as f: + f.write('dummy text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + gd._runcmd(['git', 'checkout', '-b', 'dev-branch']) + with open(os.path.join(dirname, 'foo'), 'w') as f: + f.write('updated text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Second commit']) + self.repos.append(gd) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + @staticmethod + def lrm(*args, **kwargs): + return morphlib.branchmanager.LocalRefManager(*args, **kwargs) + + def test_refs_added(self): + refinfo = [] + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + refinfo.append(commit) + lrm.add(gd, 'refs/heads/create%d' % i, commit) + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit( + 'refs/heads/create%d' % i), + refinfo[i]) + + def test_add_rollback(self): + with self.assertRaises(Exception): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.add(gd, 'refs/heads/create%d' % i, commit) + raise Exception() + for i, gd in enumerate(self.repos): + with self.assertRaises(morphlib.gitdir.InvalidRefError): + gd.resolve_ref_to_commit('refs/heads/create%d' % i) + + def test_add_rollback_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + ref = 'refs/heads/create%d' % i + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.add(gd, ref, commit) + # Make changes independent of LRM, so that rollback fails + new_commit = gd.resolve_ref_to_commit( + 'refs/heads/dev-branch') + gd.update_ref(ref, new_commit, commit) + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions], + [morphlib.gitdir.RefDeleteError] * self.REPO_COUNT) + + def test_refs_updated(self): + refinfo = [] + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + old_master = gd.resolve_ref_to_commit('refs/heads/master') + commit = gd.resolve_ref_to_commit('refs/heads/dev-branch') + refinfo.append(commit) + lrm.update(gd, 'refs/heads/master', commit, old_master) + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'), + refinfo[i]) + + def test_update_rollback(self): + refinfo = [] + with self.assertRaises(Exception): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + old_master = gd.resolve_ref_to_commit('refs/heads/master') + commit = gd.resolve_ref_to_commit('refs/heads/dev-branch') + refinfo.append(old_master) + lrm.update(gd, 'refs/heads/master', commit, old_master) + raise Exception() + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'), + refinfo[i]) + + def test_update_rollback_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + old_master = gd.resolve_ref_to_commit('refs/heads/master') + commit = gd.resolve_ref_to_commit('refs/heads/dev-branch') + lrm.update(gd, 'refs/heads/master', commit, old_master) + # Delete the ref, so rollback fails + gd.delete_ref('refs/heads/master', commit) + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions], + [morphlib.gitdir.RefUpdateError] * self.REPO_COUNT) + + def test_refs_deleted(self): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.delete(gd, 'refs/heads/master', commit) + for i, gd in enumerate(self.repos): + self.assertRaises(morphlib.gitdir.InvalidRefError, + gd.resolve_ref_to_commit, 'refs/heads/master') + + def test_delete_rollback(self): + refinfo = [] + with self.assertRaises(Exception): + with self.lrm() as lrm: + for i, gd in enumerate(self.repos): + commit = gd.resolve_ref_to_commit('refs/heads/master') + refinfo.append(commit) + lrm.delete(gd, 'refs/heads/master', commit) + raise Exception() + for i, gd in enumerate(self.repos): + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/master'), + refinfo[i]) + + def test_delete_rollback_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with self.lrm() as lrm: + for gd in self.repos: + commit = gd.resolve_ref_to_commit('refs/heads/master') + lrm.delete(gd, 'refs/heads/master', commit) + gd.add_ref('refs/heads/master', commit) + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assertEqual([e.__class__ for _, _, e in cm.exception.exceptions], + [morphlib.gitdir.RefAddError] * self.REPO_COUNT) + + +class RemoteRefManagerTests(unittest.TestCase): + + TARGET_COUNT = 2 + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.source = os.path.join(self.tempdir, 'source') + os.mkdir(self.source) + self.sgd = morphlib.gitdir.init(self.source) + with open(os.path.join(self.source, 'foo'), 'w') as f: + f.write('dummy text\n') + self.sgd._runcmd(['git', 'add', '.']) + self.sgd._runcmd(['git', 'commit', '-m', 'Initial commit']) + self.sgd._runcmd(['git', 'checkout', '-b', 'dev-branch']) + with open(os.path.join(self.source, 'foo'), 'w') as f: + f.write('updated text\n') + self.sgd._runcmd(['git', 'add', '.']) + self.sgd._runcmd(['git', 'commit', '-m', 'Second commit']) + self.sgd._runcmd(['git', 'checkout', '--orphan', 'no-ff']) + with open(os.path.join(self.source, 'foo'), 'w') as f: + f.write('parallel dimension text\n') + self.sgd._runcmd(['git', 'add', '.']) + self.sgd._runcmd(['git', 'commit', '-m', 'Non-fast-forward commit']) + + self.remotes = [] + for i in xrange(self.TARGET_COUNT): + name = 'remote-%d' % i + dirname = os.path.join(self.tempdir, name) + + # Allow deleting HEAD + cliapp.runcmd(['git', 'init', '--bare', dirname]) + gd = morphlib.gitdir.GitDirectory(dirname) + gd.set_config('receive.denyDeleteCurrent', 'warn') + + self.sgd._runcmd(['git', 'remote', 'add', name, dirname]) + self.remotes.append((name, dirname, gd)) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + @staticmethod + def list_refs(gd): + out = gd._runcmd(['git', 'for-each-ref', + '--format=%(refname)%00%(objectname)%00']) + return dict(line.split('\0') for line in + out.strip('\0\n').split('\0\n') if line) + + def push_creates(self, rrm): + for name, dirname, gd in self.remotes: + rrm.push(self.sgd.get_remote(name), + morphlib.gitdir.RefSpec('refs/heads/master'), + morphlib.gitdir.RefSpec('refs/heads/dev-branch')) + + def push_deletes(self, rrm): + null_commit = '0' * 40 + master_commit = self.sgd.resolve_ref_to_commit('refs/heads/master') + dev_commit = self.sgd.resolve_ref_to_commit('refs/heads/dev-branch') + for name, dirname, gd in self.remotes: + rrm.push(self.sgd.get_remote(name), + morphlib.gitdir.RefSpec( + source=null_commit, + target='refs/heads/master', + require=master_commit), + morphlib.gitdir.RefSpec( + source=null_commit, + target='refs/heads/dev-branch', + require=dev_commit)) + + def assert_no_remote_branches(self): + for name, dirname, gd in self.remotes: + self.assertEqual(self.list_refs(gd), {}) + + def assert_remote_branches(self): + for name, dirname, gd in self.remotes: + for name, sha1 in self.list_refs(gd).iteritems(): + self.assertEqual(self.sgd.resolve_ref_to_commit(name), sha1) + + def test_rollback_after_create_success(self): + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_creates(rrm) + self.assert_remote_branches() + self.assert_no_remote_branches() + + def test_rollback_after_create_failure(self): + failure_exception = Exception() + with self.assertRaises(Exception) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_creates(rrm) + raise failure_exception + self.assertEqual(cm.exception, failure_exception) + self.assert_no_remote_branches() + + @unittest.skip('No way to have conditional delete until Git 1.8.5') + def test_rollback_after_create_cleanup_failure(self): + failure_exception = Exception() + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_creates(rrm) + + # Break rollback with a new non-ff commit on master + no_ff = self.sgd.resolve_ref_to_commit('no-ff') + master = 'refs/heads/master' + master_commit = \ + self.sgd.resolve_ref_to_commit('refs/heads/master') + for name, dirname, gd in self.remotes: + r = self.sgd.get_remote(name) + r.push(morphlib.gitdir.RefSpec(source=no_ff, target=master, + require=master_commit, + force=True)) + + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + self.assert_no_remote_branches() + + def test_rollback_after_deletes_success(self): + for name, dirname, gd in self.remotes: + self.sgd.get_remote(name).push( + morphlib.gitdir.RefSpec('master'), + morphlib.gitdir.RefSpec('dev-branch')) + self.assert_remote_branches() + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_deletes(rrm) + self.assert_no_remote_branches() + self.assert_remote_branches() + + def test_rollback_after_deletes_failure(self): + failure_exception = Exception() + for name, dirname, gd in self.remotes: + self.sgd.get_remote(name).push( + morphlib.gitdir.RefSpec('master'), + morphlib.gitdir.RefSpec('dev-branch')) + self.assert_remote_branches() + with self.assertRaises(Exception) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_deletes(rrm) + raise failure_exception + self.assertEqual(cm.exception, failure_exception) + self.assert_remote_branches() + + def test_rollback_after_deletes_cleanup_failure(self): + failure_exception = Exception() + for name, dirname, gd in self.remotes: + self.sgd.get_remote(name).push( + morphlib.gitdir.RefSpec('master'), + morphlib.gitdir.RefSpec('dev-branch')) + with self.assertRaises(morphlib.branchmanager.RefCleanupError) as cm: + with morphlib.branchmanager.RemoteRefManager() as rrm: + self.push_deletes(rrm) + + # Break rollback with a new non-ff commit on master + no_ff = self.sgd.resolve_ref_to_commit('no-ff') + master = 'refs/heads/master' + master_commit = \ + self.sgd.resolve_ref_to_commit('refs/heads/master') + for name, dirname, gd in self.remotes: + r = self.sgd.get_remote(name) + r.push(morphlib.gitdir.RefSpec(source=no_ff, target=master, + require=master_commit)) + + raise failure_exception + self.assertEqual(cm.exception.primary_exception, failure_exception) + diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 7ee64f36..be2137b2 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -16,10 +16,10 @@ # =*= License: GPL-2 =*= -import collections import cliapp -import glob +import itertools import os +import re import morphlib @@ -39,6 +39,285 @@ class InvalidRefError(cliapp.AppException): 'at ref %s.' %(repo.dirname, ref)) +class ExpectedSha1Error(cliapp.AppException): + + def __init__(self, ref): + self.ref = ref + cliapp.AppException.__init__( + self, 'SHA1 expected, got %s' % ref) + + +class RefChangeError(cliapp.AppException): + pass + + +class RefAddError(RefChangeError): + + def __init__(self, gd, ref, sha1, original_exception): + self.gd = gd + self.dirname = dirname = gd.dirname + self.ref = ref + self.sha1 = sha1 + self.original_exception = original_exception + RefChangeError.__init__(self, 'Adding ref %(ref)s '\ + 'with commit %(sha1)s failed in git repository '\ + 'located at %(dirname)s: %(original_exception)r' % locals()) + + +class RefUpdateError(RefChangeError): + + def __init__(self, gd, ref, old_sha1, new_sha1, original_exception): + self.gd = gd + self.dirname = dirname = gd.dirname + self.ref = ref + self.old_sha1 = old_sha1 + self.new_sha1 = new_sha1 + self.original_exception = original_exception + RefChangeError.__init__(self, 'Updating ref %(ref)s '\ + 'from %(old_sha1)s to %(new_sha1)s failed in git repository '\ + 'located at %(dirname)s: %(original_exception)r' % locals()) + + +class RefDeleteError(RefChangeError): + + def __init__(self, gd, ref, sha1, original_exception): + self.gd = gd + self.dirname = dirname = gd.dirname + self.ref = ref + self.sha1 = sha1 + self.original_exception = original_exception + RefChangeError.__init__(self, 'Deleting ref %(ref)s '\ + 'expecting commit %(sha1)s failed in git repository '\ + 'located at %(dirname)s: %(original_exception)r' % locals()) + + +class InvalidRefSpecError(cliapp.AppException): + + def __init__(self, source, target): + self.source = source + self.target = target + cliapp.AppException.__init__( + self, 'source or target must be defined, '\ + 'got %(source)r and %(target)r respectively.' % locals()) + + +class PushError(cliapp.AppException): + pass + + +class NoRefspecsError(PushError): + + def __init__(self, remote): + self.remote = remote.name + PushError.__init__(self, + 'Push to remote %r was given no refspecs.' % remote) + + +class PushFailureError(PushError): + + def __init__(self, remote, refspecs, exit, results, stderr): + self.remote = remote.name + self.push_url = push_url = remote.get_push_url() + self.refspecs = refspecs + self.exit = exit + self.results = results + self.stderr = stderr + PushError.__init__(self, 'Push to remote %(remote)r, '\ + 'push url %(push_url)s '\ + 'with refspecs %(refspecs)r '\ + 'failed with exit code %(exit)s' % locals()) + + +class RefSpec(object): + '''Class representing how to push or pull a ref. + + `source` is a reference to the local commit/tag you want to push to + the remote. + `target` is the ref on the remote you want to push to. + `require` is the value that the remote is expected to currently be. + Currently `require` is only used to provide a reverse of the respec, + but future versions of Git will support requiring the value of + `target` on the remote to be at a certain commit, or fail. + `force` defaults to false, and if set adds the flag to push even if + it's non-fast-forward. + + If `source` is not provided, but `target` is, then the refspec will + delete `target` on the remote. + If `source` is provided, but `target` is not, then `source` is used + as the `target`, since if you specify a ref for the `source`, you + can push the same local branch to the same remote branch. + + ''' + + def __init__(self, source=None, target=None, require=None, force=False): + if source is None and target is None: + raise InvalidRefSpecError(source, target) + self.source = source + self.target = target + self.require = require + self.force = force + if target is None: + # Default to source if target not given, source must be a + # branch name, or when this refspec is pushed it will fail. + self.target = target = source + if source is None: # Delete if source not given + self.source = source = '0' * 40 + + @property + def push_args(self): + '''Arguments to pass to push to push this ref. + + Returns an iterable of the arguments that would need to be added + to a push command to push this ref spec. + + This currently returns a single-element tuple, but it may expand + to multiple arguments, e.g. + 1. tags expand to `tag "$name"` + 2. : expands to all the matching refs + 3. When Git 1.8.5 becomes available, + `"--force-with-lease=$target:$required" "$source:$target"`. + + ''' + + # TODO: Use require parameter when Git 1.8.5 is available, + # to allow the push to fail if the target ref is not at + # that commit by using the --force-with-lease option. + return ('%(force)s%(source)s:%(target)s' % { + 'force': '+' if self.force else '', + 'source': self.source, + 'target': self.target + }), + + def revert(self): + '''Create a respec which will undo the effect of pushing this one. + + If `require` was not specified, the revert refspec will delete + the branch. + + ''' + + return self.__class__(source=(self.require or '0' * 40), + target=self.target, require=self.source, + force=self.force) + + +PUSH_FORMAT = re.compile(r''' +# Match flag, this is the eventual result in a nutshell +(?P<flag>[- +*=!])\t +# The refspec is colon separated and separated from the rest by another tab. +(?P<from>[^:]*):(?P<to>[^\t]*)\t +# Two possible formats remain, so separate the two with a capture group +(?: + # Summary is an arbitrary string, separated from the reason by a space + (?P<summary>.*)[ ] + # Reason is enclosed in parenthesis and ends the line + \((?P<reason>.*)\) + # The reason is optional, so we may instead only have the summary + | (?P<summary_only>.*) +) +''', re.VERBOSE) + + +class Remote(object): + '''Represent a remote git repository. + + This can either be nascent or concrete, depending on whether the + name is given. + + Changes to a concrete remote's config are written-through to git's + config files, while a nascent remote keeps changes in-memory. + + ''' + + def __init__(self, gd, name=None): + self.gd = gd + self.name = name + self.push_url = None + self.fetch_url = None + + def set_fetch_url(self, url): + self.fetch_url = url + if self.name is not None: + self.gd._runcmd(['git', 'remote', 'set-url', self.name, url]) + + def set_push_url(self, url): + self.push_url = url + if self.name is not None: + self.gd._runcmd(['git', 'remote', 'set-url', '--push', + self.name, url]) + + def _get_remote_url(self, remote_name, kind): + # As distasteful as it is to parse the output of porcelain + # commands, this is the best option. + # Git config can be used to get the raw value, but this is + # incorrect when url.*.insteadof rules are involved. + # Re-implementing the rewrite logic in morph is duplicated effort + # and more work to keep it in sync. + # It's possible to get the fetch url with `git ls-remote --get-url + # <remote>`, but this will just print the remote's name if it + # is not defined. + # It is only possible to use git to get the push url by parsing + # `git remote -v` or `git remote show -n <remote>`, and `git + # remote -v` is easier to parse. + output = self.gd._runcmd(['git', 'remote', '-v']) + for line in output.splitlines(): + words = line.split() + if (len(words) == 3 and + words[0] == remote_name and + words[2] == '(%s)' % kind): + return words[1] + + return None + + def get_fetch_url(self): + if self.name is None: + return self.fetch_url + return self._get_remote_url(self.name, 'fetch') + + def get_push_url(self): + if self.name is None: + return self.push_url or self.get_fetch_url() + return self._get_remote_url(self.name, 'push') + + @staticmethod + def _parse_push_output(output): + for line in output.splitlines(): + m = PUSH_FORMAT.match(line) + # Push may output lines that are not related to the status, + # so ignore any that don't match the status format. + if m is None: + continue + # Ensure the same number of arguments + ret = list(m.group('flag', 'from', 'to')) + ret.append(m.group('summary') or m.group('summary_only')) + ret.append(m.group('reason')) + yield tuple(ret) + + def push(self, *refspecs): + '''Push given refspecs to the remote and return results. + + If no refspecs are given, an exception is raised. + + Returns an iterable of (flag, from_ref, to_ref, summary, reason) + + If the push fails, a PushFailureError is raised, from which the + result can be retrieved with the `results` field. + + ''' + + if not refspecs: + raise NoRefspecsError(self) + push_name = self.name or self.get_push_url() + cmdline = ['git', 'push', '--porcelain', push_name] + cmdline.extend(itertools.chain.from_iterable( + rs.push_args for rs in refspecs)) + exit, out, err = self.gd._runcmd_unchecked(cmdline) + if exit != 0: + raise PushFailureError(self, refspecs, exit, + self._parse_push_output(out), err) + return self._parse_push_output(out) + + class GitDirectory(object): '''Represent a git working tree + .git directory. @@ -65,6 +344,9 @@ class GitDirectory(object): return cliapp.runcmd(argv, cwd=self.dirname, **kwargs) + def _runcmd_unchecked(self, *args, **kwargs): + return cliapp.runcmd_unchecked(*args, cwd=self.dirname, **kwargs) + def checkout(self, branch_name): # pragma: no cover '''Check out a git branch.''' self._runcmd(['git', 'checkout', branch_name]) @@ -96,13 +378,33 @@ class GitDirectory(object): parsed_head = self._runcmd(['git', 'rev-parse', 'HEAD']).strip() return parsed_ref == parsed_head - def cat_file(self, obj_type, ref, filename): # pragma: no cover + def get_file_from_ref(self, ref, filename): # pragma: no cover + '''Get file contents from git by ref and filename. + + `ref` should be a tree-ish e.g. HEAD, master, refs/heads/master, + refs/tags/foo, though SHA1 tag, commit or tree IDs are also valid. + + `filename` is the path to the file object from the base of the + git directory. + + Returns the contents of the referred to file as a string. + + ''' + + # Blob ID is left as the git revision, rather than SHA1, since + # we know get_blob_contents will accept it + blob_id = '%s:%s' % (ref, filename) + return self.get_blob_contents(blob_id) + + def get_blob_contents(self, blob_id): # pragma: no cover + '''Get file contents from git by ID''' return self._runcmd( - ['git', 'cat-file', obj_type, '%s:%s' % (ref, filename)]) + ['git', 'cat-file', 'blob', blob_id]) - def update_remotes(self): # pragma: no cover - '''Update remotes.''' - self._runcmd(['git', 'remote', 'update', '--prune']) + def get_commit_contents(self, commit_id): # pragma: no cover + '''Get commit contents from git by ID''' + return self._runcmd( + ['git', 'cat-file', 'commit', commit_id]) def update_submodules(self, app): # pragma: no cover '''Change .gitmodules URLs, and checkout submodules.''' @@ -122,23 +424,17 @@ class GitDirectory(object): def get_config(self, key): '''Return value for a git repository configuration variable.''' - value = self._runcmd(['git', 'config', key]) - return value.strip() + value = self._runcmd(['git', 'config', '-z', key]) + return value.rstrip('\0') - def set_remote_fetch_url(self, remote_name, url): - '''Set the fetch URL for a remote.''' - self._runcmd(['git', 'remote', 'set-url', remote_name, url]) + def get_remote(self, *args, **kwargs): + '''Get a remote for this Repository. - def get_remote_fetch_url(self, remote_name): - '''Return the fetch URL for a given remote.''' - output = self._runcmd(['git', 'remote', '-v']) - for line in output.splitlines(): - words = line.split() - if (len(words) == 3 and - words[0] == remote_name and - words[2] == '(fetch)'): - return words[1] - return None + Gets a previously configured remote if a remote name is given. + Otherwise a nascent one is created. + + ''' + return Remote(self, *args, **kwargs) def update_remotes(self): # pragma: no cover '''Run "git remote update --prune".''' @@ -165,13 +461,18 @@ class GitDirectory(object): else: return self._list_files_in_ref(ref) - def _rev_parse_tree(self, ref): + def _rev_parse(self, ref): try: - return self._runcmd(['git', 'rev-parse', '--verify', - '%s^{tree}' % ref]).strip() + return self._runcmd(['git', 'rev-parse', '--verify', ref]).strip() except cliapp.AppException as e: raise InvalidRefError(self, ref) + def resolve_ref_to_commit(self, ref): + return self._rev_parse('%s^{commit}' % ref) + + def resolve_ref_to_tree(self, ref): + return self._rev_parse('%s^{tree}' % ref) + def _list_files_in_work_tree(self): for dirpath, subdirs, filenames in os.walk(self.dirname): if dirpath == self.dirname and '.git' in subdirs: @@ -180,7 +481,7 @@ class GitDirectory(object): yield os.path.join(dirpath, filename)[len(self.dirname)+1:] def _list_files_in_ref(self, ref): - tree = self._rev_parse_tree(ref) + tree = self.resolve_ref_to_tree(ref) output = self._runcmd(['git', 'ls-tree', '--name-only', '-rz', tree]) # ls-tree appends \0 instead of interspersing, so we need to # strip the trailing \0 before splitting @@ -193,44 +494,121 @@ class GitDirectory(object): if ref is None: with open(os.path.join(self.dirname, filename)) as f: return f.read() - tree = self._rev_parse_tree(ref) - return self.cat_file('blob', tree, filename) + tree = self.resolve_ref_to_tree(ref) + return self.get_file_from_ref(tree, filename) @property def HEAD(self): output = self._runcmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) return output.strip() - def _get_status(self): - '''Runs git status and formats its output into something more useful. + def get_index(self, index_file=None): + return morphlib.gitindex.GitIndex(self, index_file) - This runs git status such that unusual filenames are preserved - and returns its output in a sequence of (status_code, to_path, - from_path). + def store_blob(self, blob_contents): + '''Hash `blob_contents`, store it in git and return the sha1. - from_path is None unless the status_code says there was a rename, - in which case it is the path it was renamed from. + `blob_contents` must either be a string or a value suitable to + pass to subprocess.Popen i.e. a file descriptor or file object + with fileno() method. - Untracked and ignored changes are also included in the output, - their status codes are '??' and '!!' respectively. + ''' + if isinstance(blob_contents, basestring): + kwargs = {'feed_stdin': blob_contents} + else: + kwargs = {'stdin': blob_contents} + return self._runcmd(['git', 'hash-object', '-t', 'blob', + '-w', '--stdin'], **kwargs).strip() + + def commit_tree(self, tree, parent, message, **kwargs): + '''Create a commit''' + # NOTE: Will need extension for 0 or N parents. + env = {} + for who, info in itertools.product(('committer', 'author'), + ('name', 'email')): + argname = '%s_%s' % (who, info) + envname = 'GIT_%s_%s' % (who.upper(), info.upper()) + if argname in kwargs: + env[envname] = kwargs[argname] + for who in ('committer', 'author'): + argname = '%s_date' % who + envname = 'GIT_%s_DATE' % who.upper() + if argname in kwargs: + env[envname] = kwargs[argname].isoformat() + return self._runcmd(['git', 'commit-tree', tree, + '-p', parent, '-m', message], + env=env).strip() + + @staticmethod + def _check_is_sha1(string): + if not morphlib.git.is_valid_sha1(string): + raise ExpectedSha1Error(string) + + def _update_ref(self, ref_args, message): + args = ['git', 'update-ref'] + # No test coverage, since while this functionality is useful, + # morph does not need an API for inspecting the reflog, so + # it existing purely to test ref updates is a tad overkill. + if message is not None: # pragma: no cover + args.extend(('-m', message)) + args.extend(ref_args) + self._runcmd(args) + + def add_ref(self, ref, sha1, message=None): + '''Create a ref called `ref` in the repository pointing to `sha1`. + + `message` is a string to add to the reflog about this change + `ref` must not already exist, if it does, use `update_ref` + `sha1` must be a 40 character hexadecimal string representing + the SHA1 of the commit or tag this ref will point to, this is + the result of the commit_tree or resolve_ref_to_commit methods. + + ''' + self._check_is_sha1(sha1) + # 40 '0' characters is code for no previous value + # this ensures it will fail if the branch already exists + try: + return self._update_ref((ref, sha1, '0' * 40), message) + except Exception, e: + raise RefAddError(self, ref, sha1, e) + + def update_ref(self, ref, sha1, old_sha1, message=None): + '''Change the commit the ref `ref` points to, to `sha1`. + + `message` is a string to add to the reflog about this change + `sha1` and `old_sha` must be 40 character hexadecimal strings + representing the SHA1 of the commit or tag this ref will point + to and currently points to respectively. This is the result of + the commit_tree or resolve_ref_to_commit methods. + `ref` must exist, and point to `old_sha1`. + This is to avoid unexpected results when multiple processes + attempt to change refs. ''' - status = self._runcmd(['git', 'status', '-z', '--ignored']) - tokens = collections.deque(status.split('\0')) - while True: - tok = tokens.popleft() - # Terminates with an empty token, since status ends with a \0 - if not tok: - return - - code = tok[:2] - to_path = tok[3:] - yield code, to_path, tokens.popleft() if code[0] == 'R' else None - - def get_uncommitted_changes(self): - for code, to_path, from_path in self._get_status(): - if code not in ('??', '!!'): - yield code, to_path, from_path + self._check_is_sha1(sha1) + self._check_is_sha1(old_sha1) + try: + return self._update_ref((ref, sha1, old_sha1), message) + except Exception, e: + raise RefUpdateError(self, ref, old_sha1, sha1, e) + + def delete_ref(self, ref, old_sha1, message=None): + '''Remove the ref `ref`. + + `message` is a string to add to the reflog about this change + `old_sha1` must be a 40 character hexadecimal string representing + the SHA1 of the commit or tag this ref will point to, this is + the result of the commit_tree or resolve_ref_to_commit methods. + `ref` must exist, and point to `old_sha1`. + This is to avoid unexpected results when multiple processes + attempt to change refs. + + ''' + self._check_is_sha1(old_sha1) + try: + return self._update_ref(('-d', ref, old_sha1), message) + except Exception, e: + raise RefDeleteError(self, ref, old_sha1, e) def init(dirname): diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 803f1b3e..21a6b5b8 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -16,6 +16,7 @@ # =*= License: GPL-2 =*= +import datetime import os import shutil import tempfile @@ -54,15 +55,11 @@ class GitDirectoryTests(unittest.TestCase): gitdir.set_config('foo.bar', 'yoyo') self.assertEqual(gitdir.get_config('foo.bar'), 'yoyo') - def test_sets_remote(self): + def test_gets_index(self): os.mkdir(self.dirname) gitdir = morphlib.gitdir.init(self.dirname) - self.assertEqual(gitdir.get_remote_fetch_url('origin'), None) + self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex) - gitdir._runcmd(['git', 'remote', 'add', 'origin', 'foobar']) - url = 'git://git.example.com/foo' - gitdir.set_remote_fetch_url('origin', url) - self.assertEqual(gitdir.get_remote_fetch_url('origin'), url) class GitDirectoryContentsTests(unittest.TestCase): @@ -148,7 +145,307 @@ class GitDirectoryContentsTests(unittest.TestCase): gd.checkout('foo') self.assertEqual(gd.HEAD, 'foo') - def test_uncommitted_changes(self): + def test_resolve_ref(self): + # Just tests that you get an object IDs back and that the + # commit and tree IDs are different, since checking the actual + # value of the commit requires foreknowledge of the result or + # re-implementing the body in the test. + gd = morphlib.gitdir.GitDirectory(self.dirname) + commit = gd.resolve_ref_to_commit(gd.HEAD) + self.assertEqual(len(commit), 40) + tree = gd.resolve_ref_to_tree(gd.HEAD) + self.assertEqual(len(tree), 40) + self.assertNotEqual(commit, tree) + + def test_store_blob_with_string(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + sha1 = gd.store_blob('test string') + self.assertEqual('test string', gd.get_blob_contents(sha1)) + + def test_store_blob_with_file(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + with open(os.path.join(self.tempdir, 'blob'), 'w') as f: + f.write('test string') + with open(os.path.join(self.tempdir, 'blob'), 'r') as f: + sha1 = gd.store_blob(f) + self.assertEqual('test string', gd.get_blob_contents(sha1)) + + def test_commit_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + parent = gd.resolve_ref_to_commit(gd.HEAD) + tree = gd.resolve_ref_to_tree(parent) + aname = 'Author Name' + aemail = 'author@email' + cname = 'Committer Name' + cemail = 'committer@email' + pseudo_now = datetime.datetime.fromtimestamp(683074800) + + now_str = "683074800" + message= 'MESSAGE' + expected = [ + "tree %(tree)s", + "parent %(parent)s", + "author %(aname)s <%(aemail)s> %(now_str)s +0000", + "committer %(cname)s <%(cemail)s> %(now_str)s +0000", + "", + "%(message)s", + "", + ] + expected = [l % locals() for l in expected] + commit = gd.commit_tree(tree, parent, message=message, + committer_name=cname, + committer_email=cemail, + committer_date=pseudo_now, + author_name=aname, + author_email=aemail, + author_date=pseudo_now, + ) + self.assertEqual(expected, gd.get_commit_contents(commit).split('\n')) + + +class GitDirectoryRefTwiddlingTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + with open(os.path.join(self.dirname, 'foo'), 'w') as f: + f.write('dummy text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + # Add a second commit for update_ref test, so it has another + # commit to roll back from + with open(os.path.join(self.dirname, 'bar'), 'w') as f: + f.write('dummy text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Second commit']) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_expects_sha1s(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertRaises(morphlib.gitdir.ExpectedSha1Error, + gd.add_ref, 'refs/heads/foo', 'HEAD') + self.assertRaises(morphlib.gitdir.ExpectedSha1Error, + gd.update_ref, 'refs/heads/foo', 'HEAD', 'HEAD') + self.assertRaises(morphlib.gitdir.ExpectedSha1Error, + gd.update_ref, 'refs/heads/master', + gd._rev_parse(gd.HEAD), 'HEAD') + self.assertRaises(morphlib.gitdir.ExpectedSha1Error, + gd.update_ref, 'refs/heads/master', + 'HEAD', gd._rev_parse(gd.HEAD)) + self.assertRaises(morphlib.gitdir.ExpectedSha1Error, + gd.delete_ref, 'refs/heads/master', 'HEAD') + + def test_add_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + head_commit = gd.resolve_ref_to_commit(gd.HEAD) + gd.add_ref('refs/heads/foo', head_commit) + self.assertEqual(gd.resolve_ref_to_commit('refs/heads/foo'), + head_commit) + + def test_add_ref_fail(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + head_commit = gd.resolve_ref_to_commit('refs/heads/master') + self.assertRaises(morphlib.gitdir.RefAddError, + gd.add_ref, 'refs/heads/master', head_commit) + + def test_update_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + head_commit = gd._rev_parse('refs/heads/master') + prev_commit = gd._rev_parse('refs/heads/master^') + gd.update_ref('refs/heads/master', prev_commit, head_commit) + self.assertEqual(gd._rev_parse('refs/heads/master'), prev_commit) + + def test_update_ref_fail(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + head_commit = gd._rev_parse('refs/heads/master') + prev_commit = gd._rev_parse('refs/heads/master^') + gd.delete_ref('refs/heads/master', head_commit) + with self.assertRaises(morphlib.gitdir.RefUpdateError): + gd.update_ref('refs/heads/master', prev_commit, head_commit) + + def test_delete_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + head_commit = gd._rev_parse('refs/heads/master') + gd.delete_ref('refs/heads/master', head_commit) + self.assertRaises(morphlib.gitdir.InvalidRefError, + gd._rev_parse, 'refs/heads/master') + + def test_delete_ref_fail(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + prev_commit = gd._rev_parse('refs/heads/master^') + with self.assertRaises(morphlib.gitdir.RefDeleteError): + gd.delete_ref('refs/heads/master', prev_commit) + + +class GitDirectoryRemoteConfigTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_sets_urls(self): + os.mkdir(self.dirname) + gitdir = morphlib.gitdir.init(self.dirname) + remote = gitdir.get_remote('origin') + self.assertEqual(remote.get_fetch_url(), None) + self.assertEqual(remote.get_push_url(), None) + + gitdir._runcmd(['git', 'remote', 'add', 'origin', 'foobar']) + fetch_url = 'git://git.example.com/foo.git' + push_url = 'ssh://git@git.example.com/foo.git' + remote.set_fetch_url(fetch_url) + remote.set_push_url(push_url) + self.assertEqual(remote.get_fetch_url(), fetch_url) + self.assertEqual(remote.get_push_url(), push_url) + + def test_nascent_remote_fetch(self): + os.mkdir(self.dirname) + gitdir = morphlib.gitdir.init(self.dirname) + remote = gitdir.get_remote(None) + self.assertEqual(remote.get_fetch_url(), None) + self.assertEqual(remote.get_push_url(), None) + + fetch_url = 'git://git.example.com/foo.git' + push_url = 'ssh://git@git.example.com/foo.git' + remote.set_fetch_url(fetch_url) + remote.set_push_url(push_url) + self.assertEqual(remote.get_fetch_url(), fetch_url) + self.assertEqual(remote.get_push_url(), push_url) + + +class RefSpecTests(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + @staticmethod + def refspec(*args, **kwargs): + return morphlib.gitdir.RefSpec(*args, **kwargs) + + def test_input(self): + with self.assertRaises(morphlib.gitdir.InvalidRefSpecError): + morphlib.gitdir.RefSpec() + + def test_rs_from_source(self): + rs = self.refspec(source='master') + self.assertEqual(rs.push_args, ('master:master',)) + + def test_rs_from_target(self): + rs = self.refspec(target='master') + self.assertEqual(rs.push_args, ('%s:master' % ('0' * 40),)) + + def test_rs_with_target_and_source(self): + rs = self.refspec(source='foo', target='master') + self.assertEqual(rs.push_args, ('foo:master',)) + + def test_rs_with_source_and_force(self): + rs = self.refspec('master', force=True) + self.assertEqual(rs.push_args, ('+master:master',)) + + def test_rs_revert_from_source(self): + revert = self.refspec(source='master').revert() + self.assertEqual(revert.push_args, ('%s:master' % ('0' * 40),)) + + def test_rs_revert_inc_require(self): + revert = self.refspec(source='master', require=('beef'*5)).revert() + self.assertEqual(revert.push_args, ('%s:master' % ('beef' * 5),)) + + def test_rs_double_revert(self): + rs = self.refspec(target='master').revert().revert() + self.assertEqual(rs.push_args, ('%s:master' % ('0' * 40),)) + + +class GitDirectoryRemotePushTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + with open(os.path.join(self.dirname, 'foo'), 'w') as f: + f.write('dummy text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + gd._runcmd(['git', 'checkout', '-b', 'foo']) + with open(os.path.join(self.dirname, 'foo'), 'w') as f: + f.write('updated text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Second commit']) + self.mirror = os.path.join(self.tempdir, 'mirror') + gd._runcmd(['git', 'init', '--bare', self.mirror]) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_push_needs_refspecs(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + r = gd.get_remote() + r.set_push_url(self.mirror) + self.assertRaises(morphlib.gitdir.NoRefspecsError, r.push) + + def test_push_new(self): + push_master = morphlib.gitdir.RefSpec('master') + gd = morphlib.gitdir.GitDirectory(self.dirname) + r = gd.get_remote() + r.set_push_url(self.mirror) + self.assertEqual(sorted(r.push(push_master)), + [('*', 'refs/heads/master', 'refs/heads/master', + '[new branch]', None)]) + + def test_double_push(self): + push_master = morphlib.gitdir.RefSpec('master') + gd = morphlib.gitdir.GitDirectory(self.dirname) + r = gd.get_remote() + r.set_push_url(self.mirror) + r.push(push_master) + self.assertEqual(sorted(r.push(push_master)), + [('=', 'refs/heads/master', 'refs/heads/master', + '[up to date]', None)]) + + def test_push_update(self): + push_master = morphlib.gitdir.RefSpec('master') + push_foo = morphlib.gitdir.RefSpec(source='foo', target='master') + gd = morphlib.gitdir.GitDirectory(self.dirname) + r = gd.get_remote() + r.set_push_url(self.mirror) + r.push(push_master) + flag, ref_from, ref_to, summary, reason = \ + list(r.push(push_foo))[0] + self.assertEqual((flag, ref_from, ref_to), + (' ', 'refs/heads/foo', 'refs/heads/master')) + + def test_rewind_fail(self): + push_master = morphlib.gitdir.RefSpec('master') + push_foo = morphlib.gitdir.RefSpec(source='foo', target='master') + gd = morphlib.gitdir.GitDirectory(self.dirname) + r = gd.get_remote() + r.set_push_url(self.mirror) + r.push(push_foo) + with self.assertRaises(morphlib.gitdir.PushFailureError) as push_fail: + r.push(push_master) + self.assertEqual(sorted(push_fail.exception.results), + [('!', 'refs/heads/master', 'refs/heads/master', + '[rejected]', 'non-fast-forward')]) + + def test_force_push(self): + push_master = morphlib.gitdir.RefSpec('master', force=True) + push_foo = morphlib.gitdir.RefSpec(source='foo', target='master') gd = morphlib.gitdir.GitDirectory(self.dirname) - self.assertEqual(sorted(gd.get_uncommitted_changes()), - [(' D', 'foo', None)]) + r = gd.get_remote() + r.set_push_url(self.mirror) + r.push(push_foo) + flag, ref_from, ref_to, summary, reason = \ + list(r.push(push_master))[0] + self.assertEqual((flag, ref_from, ref_to, reason), + ('+', 'refs/heads/master', 'refs/heads/master', + 'forced update')) diff --git a/morphlib/gitindex.py b/morphlib/gitindex.py new file mode 100644 index 00000000..978ea0e2 --- /dev/null +++ b/morphlib/gitindex.py @@ -0,0 +1,159 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import collections + +import morphlib + + +STATUS_UNTRACKED = '??' +STATUS_IGNORED = '!!' + + +class GitIndex(object): + '''An object that represents operations on the working tree. + + Index objects can be constructed with a different path to the + index file, which can be used to construct commits without + altering the working tree, index or HEAD. + + The file must either be a previously initialised index, or a + non-existant file. + + Git creates a lock file and atomically alters the index by + renaming a temporary file into place, so `index_file` must be + in a writable directory. + + ''' + + def __init__(self, gd, index_file): + self._gd = gd + self._index_file = index_file + + def _run_git(self, *args, **kwargs): + if self._index_file is not None: + kwargs['env'] = kwargs.get('env', {}) + kwargs['env']['GIT_INDEX_FILE'] = self._index_file + return self._gd._runcmd(['git'] + list(args), **kwargs) + + def _get_status(self): + '''Return git status output in a Python useful format + + This runs git status such that unusual filenames are preserved + and returns its output in a sequence of (status_code, to_path, + from_path). + + from_path is None unless the status_code says there was a + rename, in which case it is the path it was renamed from. + + Untracked and ignored changes are also included in the output, + their status codes are '??' and '!!' respectively. + + ''' + + # git status -z will NUL terminate paths, so we don't have to + # unescape the paths it outputs. Unfortunately each status entry + # can have 1 or 2 paths, so extra parsing is required. + # To handle this, we split it into NUL delimited tokens. + # The first token of an entry is the 2 character status code, + # a space, then the path. + # If our status code starts with R then it's a rename, hence + # has a second path, requiring us to pop an extra token. + status = self._run_git('status', '-z', '--ignored') + tokens = collections.deque(status.split('\0')) + while True: + tok = tokens.popleft() + # Status output is NUL terminated rather than delimited, + # and split is for delimited output. A side effect of this is + # that we get an empty token as the last output. This suits + # us fine, as it gives us a sentinel value to terminate with. + if not tok: + return + + # The first token of an entry is 2 character status, a space, + # then the path + code = tok[:2] + to_path = tok[3:] + + # If the code starts with R then it's a rename, and + # the next token says where the file was renamed from + from_path = tokens.popleft() if code[0] == 'R' else None + yield code, to_path, from_path + + def get_uncommitted_changes(self): + for code, to_path, from_path in self._get_status(): + if code not in (STATUS_UNTRACKED, STATUS_IGNORED): + yield code, to_path, from_path + + def set_to_tree(self, treeish): + '''Modify the index to contain the contents of the treeish.''' + self._run_git('read-tree', treeish) + + def add_files_from_index_info(self, infos): + '''Add files without interacting with the working tree. + + `infos` is an iterable of (file mode string, object sha1, path) + There are no constraints on the size of the iterable + + ''' + + # update-index may take NUL terminated input lines of the entries + # to add so we generate a string for the input, rather than + # having many command line arguments, since for a large amount + # of entries, this can be too many arguments to process and the + # exec will fail. + # Generating the input as a string uses more memory than using + # subprocess.Popen directly and using .communicate, but is much + # less verbose. + feed_stdin = '\0'.join('%o %s\t%s' % (mode, sha1, path) + for mode, sha1, path in infos) + '\0' + self._run_git('update-index', '--add', '-z', '--index-info', + feed_stdin=feed_stdin) + + def add_files_from_working_tree(self, paths): + '''Add existing files to the index. + + Given an iterable of paths to files in the working tree, + relative to the git repository's top-level directory, + add the contents of the files to git's object store, + and the index. + + This is similar to the following: + + gd = GitDirectory(...) + idx = gd.get_index() + for path in paths: + fullpath = os.path.join(gd,dirname, path) + with open(fullpath, 'r') as f: + sha1 = gd.store_blob(f) + idx.add_files_from_index_info([(os.stat(fullpath).st_mode, + sha1, path)]) + + ''' + + if self._gd.is_bare(): + raise morphlib.gitdir.NoWorkingTreeError(self._gd) + # Handle paths in smaller chunks, so that the runcmd + # cannot fail from exceeding command line length + # 50 is an arbitrary limit + for paths in morphlib.util.iter_trickle(paths, 50): + self._run_git('add', *paths) + + def write_tree(self): + '''Transform the index into a tree in the object store.''' + return self._run_git('write-tree').strip() diff --git a/morphlib/gitindex_tests.py b/morphlib/gitindex_tests.py new file mode 100644 index 00000000..7a8953f2 --- /dev/null +++ b/morphlib/gitindex_tests.py @@ -0,0 +1,92 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class GitIndexTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + with open(os.path.join(self.dirname, 'foo'), 'w') as f: + f.write('dummy text\n') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + self.mirror = os.path.join(self.tempdir, 'mirror') + gd._runcmd(['git', 'clone', '--mirror', self.dirname, self.mirror]) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_uncommitted_changes(self): + idx = morphlib.gitdir.GitDirectory(self.dirname).get_index() + self.assertEqual(list(idx.get_uncommitted_changes()), []) + os.unlink(os.path.join(self.dirname, 'foo')) + self.assertEqual(sorted(idx.get_uncommitted_changes()), + [(' D', 'foo', None)]) + + def test_uncommitted_alt_index(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index(os.path.join(self.tempdir, 'index')) + self.assertEqual(sorted(idx.get_uncommitted_changes()), + [('D ', 'foo', None)]) + # 'D ' means not in the index, but in the working tree + + def test_set_to_tree_alt_index(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index(os.path.join(self.tempdir, 'index')) + # Read the HEAD commit into the index, which is the same as the + # working tree, so there are no uncommitted changes reported + # by status + idx.set_to_tree(gd.HEAD) + self.assertEqual(list(idx.get_uncommitted_changes()),[]) + + def test_add_files_from_index_info(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index(os.path.join(self.tempdir, 'index')) + filepath = os.path.join(gd.dirname, 'foo') + with open(filepath, 'r') as f: + sha1 = gd.store_blob(f) + idx.add_files_from_index_info( + [(os.stat(filepath).st_mode, sha1, 'foo')]) + self.assertEqual(list(idx.get_uncommitted_changes()),[]) + + def test_add_files_from_working_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index() + idx.add_files_from_working_tree(['foo']) + self.assertEqual(list(idx.get_uncommitted_changes()),[]) + + def test_add_files_from_working_tree_fails_in_bare(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + idx = gd.get_index() + self.assertRaises(morphlib.gitdir.NoWorkingTreeError, + idx.add_files_from_working_tree, ['foo']) + + def test_write_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + idx = gd.get_index() + self.assertEqual(idx.write_tree(), gd.resolve_ref_to_tree(gd.HEAD)) diff --git a/morphlib/morphset.py b/morphlib/morphset.py index 9ef1e804..6aabbde5 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -124,7 +124,7 @@ class MorphologySet(object): raise ChunkNotInStratumError(stratum_morph['name'], chunk_name) return repo_url, ref, morph - def _traverse_specs(self, cb_process, cb_filter=lambda s: True): + def traverse_specs(self, cb_process, cb_filter=lambda s: True): '''Higher-order function for processing every spec. This traverses every spec in all the morphologies, so all chunk, @@ -202,7 +202,7 @@ class MorphologySet(object): spec['ref'] = new_ref return True - self._traverse_specs(process_spec, wanted_spec) + self.traverse_specs(process_spec, wanted_spec) def list_refs(self): '''Return a set of all the (repo, ref) pairs in the MorphologySet. @@ -220,7 +220,7 @@ class MorphologySet(object): known.add((spec['repo'], spec['ref'])) return False - self._traverse_specs(process_spec, wanted_spec) + self.traverse_specs(process_spec, wanted_spec) return known @@ -242,7 +242,7 @@ class MorphologySet(object): spec['ref'] = new_ref return True - self._traverse_specs(process_spec, wanted_spec) + self.traverse_specs(process_spec, wanted_spec) def petrify_chunks(self, resolutions): '''Update _every_ chunk's ref to the value resolved in resolutions. @@ -272,7 +272,7 @@ class MorphologySet(object): spec['ref'] = resolutions[tup] return True - self._traverse_specs(process_chunk_spec, wanted_chunk_spec) + self.traverse_specs(process_chunk_spec, wanted_chunk_spec) def unpetrify_all(self): '''If a spec is petrified, unpetrify it. @@ -286,4 +286,4 @@ class MorphologySet(object): spec['ref'] = spec.pop('unpetrify-ref') return True - self._traverse_specs(process_spec, wanted_spec) + self.traverse_specs(process_spec, wanted_spec) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 9c4cd53e..8ad9effd 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -305,9 +305,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): def _load_morphology_from_git(self, loader, gd, ref, filename): try: - text = gd.cat_file('blob', ref, filename) + text = gd.get_file_from_ref(ref, filename) except cliapp.AppException: - text = gd.cat_file('blob', 'origin/%s' % ref, filename) + text = gd.get_file_from_ref('origin/%s' % ref, filename) return loader.load_from_string(text, filename) def _load_stratum_morphologies(self, loader, sb, system_morph): @@ -351,18 +351,6 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): logging.debug('All strata loaded') return morphset - def _invent_new_branch(self, cached_repo, default_name): - counter = 0 - candidate = default_name - while True: - try: - cached_repo.resolve_ref(candidate) - except morphlib.cachedrepo.InvalidReferenceError: - return candidate - else: - counter += 1 - candidate = '%s-%s' % (default_name, counter) - def edit(self, args): '''Edit or checkout a component in a system branch. @@ -793,7 +781,7 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): if head != branch: self.app.output.write( ' %s: unexpected ref checked out %r\n' % (repo, head)) - if any(gd.get_uncommitted_changes()): + if any(gd.get_index().get_uncommitted_changes()): has_uncommitted_changes = True self.app.output.write(' %s: uncommitted changes\n' % repo) diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index a05ca52e..73a07d5e 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -141,7 +141,8 @@ class SystemBranchDirectory(object): # and not the locally cached copy. resolver = morphlib.repoaliasresolver.RepoAliasResolver( cached_repo.app.settings['repo-alias']) - gd.set_remote_fetch_url('origin', resolver.pull_url(cached_repo.url)) + remote = gd.get_remote('origin') + remote.set_fetch_url(resolver.pull_url(cached_repo.url)) gd.set_config( 'url.%s.pushInsteadOf' % resolver.push_url(cached_repo.original_name), diff --git a/morphlib/util.py b/morphlib/util.py index 04df0633..dd2d05e1 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -13,6 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import itertools import os import re @@ -412,3 +413,12 @@ def get_host_architecture(): # pragma: no cover def sanitize_environment(env): for k in env: env[k] = str(env[k]) + +def iter_trickle(iterable, limit): + '''Split an iterable up into `limit` length chunks.''' + it = iter(iterable) + while True: + buf = list(itertools.islice(it, limit)) + if len(buf) == 0: + break + yield buf diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 2ad9e8aa..fbf7f27b 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -110,3 +110,14 @@ class ParseEnvironmentPairsTests(unittest.TestCase): d = { 'a': 1 } morphlib.util.sanitize_environment(d) self.assertTrue(isinstance(d['a'], str)) + +class IterTrickleTests(unittest.TestCase): + + def test_splits(self): + self.assertEqual(list(morphlib.util.iter_trickle("foobarbazqux", 3)), + [["f", "o", "o"], ["b", "a", "r"], + ["b", "a", "z"], ["q", "u", "x"]]) + + def test_truncated_final_sequence(self): + self.assertEqual(list(morphlib.util.iter_trickle("barquux", 3)), + [["b", "a", "r"], ["q", "u", "u"], ["x"]]) diff --git a/scripts/check-silliness b/scripts/check-silliness new file mode 100755 index 00000000..f956e647 --- /dev/null +++ b/scripts/check-silliness @@ -0,0 +1,63 @@ +#!/bin/sh +# +# Does the file contain any of the code constructs deemed silly? +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +errors=0 + +for x; do + if tr -cd '\t' < "$x" | grep . > /dev/null + then + echo "ERROR: $x contains TAB characters" 1>&2 + grep -n -F "$(printf "\t")" "$x" 1>&2 + errors=1 + fi + + case "$x" in + # Excluding yarn files since it's not possible to split up the + # IMPLEMENTS lines of them + *.yarn) ;; + *) + if awk 'length > 79' "$x" | grep . > /dev/null + then + echo "ERROR: $x has lines longer than 79 chars" 1>&2 + awk 'length > 79 { print NR, $0 }' "$x" 1>&2 + errors=1 + fi + ;; + esac + + case "$x" in + *.py) + if head -1 "$x" | grep '^#!' > /dev/null + then + echo "ERROR: $x has a hashbang" 1>&2 + errors=1 + fi + if [ -x "$x" ]; then + echo "ERROR: $x is executable" 1>&2 + errors=1 + fi + if grep except: "$x" + then + echo "ERROR: $x has a bare except:" 1>&2 + errors=1 + fi + ;; + esac +done +exit "$errors" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index b11ddab1..2981e6d9 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -34,6 +34,9 @@ run_morph() --cachedir-min-space=0 --tempdir-min-space=0 \ --no-default-config --config "$DATADIR/morph.conf" "$@" \ 2> "$DATADIR/result-$1" + local exit_code="$?" + cat "$DATADIR/result-$1" >&2 + return "$exit_code" } |