diff options
-rw-r--r-- | morphlib/buildcommand.py | 4 | ||||
-rw-r--r-- | morphlib/builder2_tests.py | 6 | ||||
-rw-r--r-- | morphlib/cachedrepo.py | 150 | ||||
-rw-r--r-- | morphlib/cachedrepo_tests.py | 180 | ||||
-rw-r--r-- | morphlib/git.py | 9 | ||||
-rw-r--r-- | morphlib/gitdir.py | 44 | ||||
-rw-r--r-- | morphlib/gitdir_tests.py | 27 | ||||
-rw-r--r-- | morphlib/localrepocache.py | 8 | ||||
-rw-r--r-- | morphlib/localrepocache_tests.py | 7 | ||||
-rw-r--r-- | morphlib/morphologyfactory.py | 5 | ||||
-rw-r--r-- | morphlib/morphologyfactory_tests.py | 26 | ||||
-rw-r--r-- | morphlib/plugins/artifact_inspection_plugin.py | 4 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 4 | ||||
-rw-r--r-- | morphlib/sourceresolver.py | 6 | ||||
-rw-r--r-- | tests.branching/branch-cleans-up-on-failure.stderr | 2 | ||||
-rw-r--r-- | tests.branching/checkout-cleans-up-on-failure.stderr | 2 | ||||
-rw-r--r-- | tests.build/missing-ref.stderr | 2 |
17 files changed, 199 insertions, 287 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index a658fc55..645a336c 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -389,13 +389,13 @@ class BuildCommand(object): source.repo = self.lrc.get_repo(repo_name) try: sha1 = source.sha1 - source.repo.resolve_ref(sha1) + source.repo.resolve_ref_to_commit(sha1) self.app.status(msg='Not updating git repository ' '%(repo_name)s because it ' 'already contains sha1 %(sha1)s', chatty=True, repo_name=repo_name, sha1=sha1) - except morphlib.cachedrepo.InvalidReferenceError: + except morphlib.gitdir.InvalidRefError: self.app.status(msg='Updating %(repo_name)s', repo_name=repo_name) source.repo.update() diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index 4fd0807a..f7a761f2 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -20,6 +20,7 @@ import StringIO import unittest import morphlib +import morphlib.gitdir_tests class FakeBuildSystem(object): @@ -50,8 +51,9 @@ class FakeSource(object): } self.name = 'a' - self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', - 'url', 'path') + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + self.repo = morphlib.cachedrepo.CachedRepo( + FakeApp(), 'repo', 'url', 'path') self.repo_name = 'url' self.original_ref = 'e' self.sha1 = 'f' diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index aad3d84e..aa2b5af1 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -15,26 +15,11 @@ import cliapp -import logging import os import morphlib -class InvalidReferenceError(cliapp.AppException): - - def __init__(self, repo, ref): - cliapp.AppException.__init__( - self, 'Ref %s is an invalid reference for repo %s' % (ref, repo)) - - -class UnresolvedNamedReferenceError(cliapp.AppException): - - def __init__(self, repo, ref): - cliapp.AppException.__init__( - self, 'Ref %s is not a SHA1 ref for repo %s' % (ref, repo)) - - class CheckoutDirectoryExistsError(cliapp.AppException): def __init__(self, repo, target_dir): @@ -105,63 +90,52 @@ class CachedRepo(object): self.is_mirror = not url.startswith('file://') self.already_updated = False - def ref_exists(self, ref): - '''Returns True if the given ref exists in the repo''' + self._gitdir = morphlib.gitdir.GitDirectory(path) - try: - self._rev_parse(ref) - except cliapp.AppException: - return False - return True + def ref_exists(self, ref): # pragma: no cover + '''Returns True if the given ref exists in the repo''' + return self._gitdir.ref_exists(ref) - def resolve_ref(self, ref): - '''Attempts to resolve a ref into its SHA1 and tree SHA1. + def resolve_ref_to_commit(self, ref): # pragma: no cover + '''Resolve a named ref to a commit SHA1. - Raises an InvalidReferenceError if the ref is not found in the - repository. + Raises gitdir.InvalidRefError if the ref does not exist. ''' + return self._gitdir.resolve_ref_to_commit(ref) - try: - absref = self._rev_parse(ref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + def resolve_ref_to_tree(self, ref): # pragma: no cover + '''Resolve a named ref to a tree SHA1. - try: - tree = self._show_tree_hash(absref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + Raises gitdir.InvalidRefError if the ref does not exist. - return absref, tree + ''' + return self._gitdir.resolve_ref_to_tree(ref) - def cat(self, ref, filename): - '''Attempts to read a file given a SHA1 ref. + def read_file(self, filename, ref): # pragma: no cover + '''Attempts to read a file from a given ref. - Raises an UnresolvedNamedReferenceError if the ref is not a SHA1 - ref. Raises an InvalidReferenceError if the SHA1 ref is not found - in the repository. Raises an IOError if the requested file is not - found in the ref. + Raises a gitdir.InvalidRefError if the ref is not found in the + repository. Raises an IOError if the requested file is not found in + the ref. ''' + return self._gitdir.read_file(filename, ref) - if not morphlib.git.is_valid_sha1(ref): - raise UnresolvedNamedReferenceError(self, ref) - try: - sha1 = self._rev_parse(ref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) + def list_files(self, ref, recurse=True): # pragma: no cover + '''Return filenames found in the tree pointed to by the given ref. - try: - return self._cat_file(sha1, filename) - except cliapp.AppException: - raise IOError('File %s does not exist in ref %s of repo %s' % - (filename, ref, self)) + Returns a gitdir.InvalidRefError if the ref is not found in the + repository. + + ''' + return self._gitdir.list_files(ref, recurse) def clone_checkout(self, ref, target_dir): '''Clone from the cache into the target path and check out a given ref. Raises a CheckoutDirectoryExistsError if the target - directory already exists. Raises an InvalidReferenceError if the + directory already exists. Raises a gitdir.InvalidRefError if the ref is not found in the repository. Raises a CheckoutError if something else goes wrong while copying the repository or checking out the SHA1 ref. @@ -171,14 +145,14 @@ class CachedRepo(object): if os.path.exists(target_dir): raise CheckoutDirectoryExistsError(self, target_dir) - self.resolve_ref(ref) + self._gitdir.resolve_ref_to_commit(ref) self._clone_into(target_dir, ref) def checkout(self, ref, target_dir): '''Unpacks the repository in a directory and checks out a commit ref. - Raises an InvalidReferenceError if the ref is not found in the + Raises an gitdir.InvalidRefError if the ref is not found in the repository. Raises a CopyError if something goes wrong with the copy of the repository. Raises a CheckoutError if something else goes wrong while copying the repository or checking out the SHA1 ref. @@ -193,25 +167,7 @@ class CachedRepo(object): # take care to turn the copy into something as good as a real clone. self._copy_repository(self.path, target_dir) - self._checkout_ref(ref, target_dir) - - def ls_tree(self, ref): - '''Return file names found in root tree. Does not recurse to subtrees. - - Raises an UnresolvedNamedReferenceError if the ref is not a SHA1 - ref. Raises an InvalidReferenceError if the SHA1 ref is not found - in the repository. - - ''' - - if not morphlib.git.is_valid_sha1(ref): - raise UnresolvedNamedReferenceError(self, ref) - try: - sha1 = self._rev_parse(ref) - except cliapp.AppException: - raise InvalidReferenceError(self, ref) - - return self._ls_tree(sha1) + self._checkout_ref_in_clone(ref, target_dir) def requires_update_for_ref(self, ref): '''Returns False if there's no need to update this cached repo. @@ -232,7 +188,7 @@ class CachedRepo(object): # Named refs that are valid SHA1s will confuse this code. ref_can_change = not morphlib.git.is_valid_sha1(ref) - if ref_can_change or not self.ref_exists(ref): + if ref_can_change or not self._gitdir.ref_exists(ref): return True else: return False @@ -249,9 +205,10 @@ class CachedRepo(object): return try: - self._update() + self._gitdir.update_remotes( + echo_stderr=self.app.settings['verbose']) self.already_updated = True - except cliapp.AppException, e: + except cliapp.AppException: raise UpdateError(self) def _runcmd(self, *args, **kwargs): # pragma: no cover @@ -259,27 +216,11 @@ class CachedRepo(object): kwargs['cwd'] = self.path return self.app.runcmd(*args, **kwargs) - def _rev_parse(self, ref): # pragma: no cover - return morphlib.git.gitcmd(self._runcmd, 'rev-parse', '--verify', - '%s^{commit}' % ref)[0:40] - - def _show_tree_hash(self, absref): # pragma: no cover - return morphlib.git.gitcmd(self._runcmd, 'rev-parse', '--verify', - '%s^{tree}' % absref).strip() - - def _ls_tree(self, ref): # pragma: no cover - result = morphlib.git.gitcmd(self._runcmd, 'ls-tree', - '--name-only', ref) - return result.split('\n') - - def _cat_file(self, ref, filename): # pragma: no cover - return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', - '%s:%s' % (ref, filename)) - - def _clone_into(self, target_dir, ref): #pragma: no cover + def _clone_into(self, target_dir, ref): # pragma: no cover '''Actually perform the clone''' try: - morphlib.git.clone_into(self._runcmd, self.path, target_dir, ref) + morphlib.git.clone_into(self._runcmd, self.path, target_dir, + ref) except cliapp.AppException: raise CloneError(self, target_dir) @@ -290,16 +231,15 @@ class CachedRepo(object): except cliapp.AppException: raise CopyError(self, target_dir) - def _checkout_ref(self, ref, target_dir): # pragma: no cover + def _checkout_ref_in_clone(self, ref, clone_dir): # pragma: no cover + # This is a separate GitDirectory instance. Don't confuse it with the + # internal ._gitdir attribute! + working_gitdir = morphlib.gitdir.GitDirectory(clone_dir) try: - morphlib.git.checkout_ref(self._runcmd, target_dir, ref) - except cliapp.AppException: - raise CheckoutError(self, ref, target_dir) - - def _update(self): # pragma: no cover - morphlib.git.gitcmd(self._runcmd, 'remote', 'update', - 'origin', '--prune', - echo_stderr=self.app.settings['verbose']) + working_gitdir.checkout(ref) + except cliapp.AppException as e: + raise CheckoutError(self, ref, clone_dir) + return working_gitdir def __str__(self): # pragma: no cover return self.url diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index d3ae331a..6f87bfdd 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -22,14 +22,18 @@ import fs.tempfs import cliapp import morphlib +import morphlib.gitdir_tests -class CachedRepoTests(unittest.TestCase): +class FakeApplication(object): + + def __init__(self): + self.settings = { + 'verbose': True + } + - EXAMPLE_MORPH = '''{ - "name": "foo", - "kind": "chunk" - }''' +class CachedRepoTests(unittest.TestCase): known_commit = 'a4da32f5a81c8bc6d660404724cedc3bc0914a75' bad_sha1_known_to_rev_parse = 'cafecafecafecafecafecafecafecafecafecafe' @@ -44,85 +48,43 @@ class CachedRepoTests(unittest.TestCase): 'master': 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', 'baserock/morph': '8b780e2e6f102fcf400ff973396566d36d730501' } + ref = ref.rstrip('^{commit}') try: return output[ref] except KeyError: raise cliapp.AppException('git rev-parse --verify %s' % ref) - def show_tree_hash(self, absref): - output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': - 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', - '8b780e2e6f102fcf400ff973396566d36d730501': - 'ffffffffffffffffffffffffffffffffffffffff', - 'a4da32f5a81c8bc6d660404724cedc3bc0914a75': - 'dddddddddddddddddddddddddddddddddddddddd' - } - try: - return output[absref] - except KeyError: - raise cliapp.AppException('git log -1 --format=format:%%T %s' % - absref) - - def cat_file(self, ref, filename): - output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9:foo.morph': - self.EXAMPLE_MORPH - } - try: - return output['%s:%s' % (ref, filename)] - except KeyError: - raise cliapp.AppException( - 'git cat-file blob %s:%s' % (ref, filename)) - def copy_repository(self, source_dir, target_dir): if target_dir.endswith('failed-checkout'): raise morphlib.cachedrepo.CopyError(self.repo, target_dir) def checkout_ref(self, ref, target_dir): - if ref == 'a4da32f5a81c8bc6d660404724cedc3bc0914a75': - raise morphlib.cachedrepo.CloneError(self.repo, target_dir) - elif ref == '079bbfd447c8534e464ce5d40b80114c2022ebf4': + if ref == '079bbfd447c8534e464ce5d40b80114c2022ebf4': raise morphlib.cachedrepo.CheckoutError(self.repo, ref, target_dir) else: with open(os.path.join(target_dir, 'foo.morph'), 'w') as f: f.write('contents of foo.morph') - def ls_tree(self, ref): - output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': - ['foo.morph'] - } - try: - return output[ref] - except KeyError: - raise cliapp.AppException('git ls-tree --name-only %s' % (ref)) - def clone_into(self, target_dir, ref): if target_dir.endswith('failed-checkout'): raise morphlib.cachedrepo.CloneError(self.repo, target_dir) self.clone_target = target_dir self.clone_ref = ref - def update_successfully(self): + def update_successfully(self, **kwargs): pass - def update_with_failure(self): + def update_with_failure(self, **kwargs): raise cliapp.AppException('git remote update origin') def setUp(self): self.repo_name = 'foo' self.repo_url = 'git://foo.bar/foo.git' self.repo_path = '/tmp/foo' - self.repo = morphlib.cachedrepo.CachedRepo( - object(), self.repo_name, self.repo_url, self.repo_path) - self.repo._rev_parse = self.rev_parse - self.repo._show_tree_hash = self.show_tree_hash - self.repo._cat_file = self.cat_file - self.repo._copy_repository = self.copy_repository - self.repo._checkout_ref = self.checkout_ref - self.repo._ls_tree = self.ls_tree - self.repo._clone_into = self.clone_into + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + self.repo = morphlib.cachedrepo.CachedRepo( + FakeApplication(), self.repo_name, self.repo_url, + self.repo_path) self.tempfs = fs.tempfs.TempFS() def test_constructor_sets_name_and_url_and_path(self): @@ -130,87 +92,47 @@ class CachedRepoTests(unittest.TestCase): self.assertEqual(self.repo.url, self.repo_url) self.assertEqual(self.repo.path, self.repo_path) - def test_ref_exists(self): - self.assertEqual(self.repo.ref_exists('master'), True) - - def test_ref_does_not_exist(self): - self.assertEqual(self.repo.ref_exists('non-existant-ref'), False) - - def test_resolve_named_ref_master(self): - sha1, tree = self.repo.resolve_ref('master') - self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee') - - def test_resolve_named_ref_baserock_morph(self): - sha1, tree = self.repo.resolve_ref('baserock/morph') - self.assertEqual(sha1, '8b780e2e6f102fcf400ff973396566d36d730501') - self.assertEqual(tree, 'ffffffffffffffffffffffffffffffffffffffff') - - def test_fail_resolving_invalid_named_ref(self): - self.assertRaises(morphlib.cachedrepo.InvalidReferenceError, - self.repo.resolve_ref, 'foo/bar') - - def test_resolve_sha1_ref(self): - sha1, tree = self.repo.resolve_ref( - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee') - - def test_fail_resolving_an_invalid_sha1_ref(self): - self.assertRaises(morphlib.cachedrepo.InvalidReferenceError, - self.repo.resolve_ref, - self.bad_sha1_known_to_rev_parse) - - def test_cat_existing_file_in_existing_ref(self): - data = self.repo.cat('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', - 'foo.morph') - self.assertEqual(data, self.EXAMPLE_MORPH) - - def test_fail_cat_file_in_invalid_ref(self): - self.assertRaises( - morphlib.cachedrepo.InvalidReferenceError, self.repo.cat, - '079bbfd447c8534e464ce5d40b80114c2022ebf4', - 'doesnt-matter-whether-this-file-exists') - - def test_fail_cat_non_existent_file_in_existing_ref(self): - self.assertRaises(IOError, self.repo.cat, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', - 'file-that-does-not-exist') - - def test_fail_cat_non_existent_file_in_invalid_ref(self): - self.assertRaises( - morphlib.cachedrepo.InvalidReferenceError, self.repo.cat, - '079bbfd447c8534e464ce5d40b80114c2022ebf4', - 'file-that-does-not-exist') - - def test_fail_because_cat_in_named_ref_is_not_allowed(self): - self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError, - self.repo.cat, 'master', 'doesnt-matter') - def test_fail_clone_checkout_into_existing_directory(self): + self.repo._gitdir.checkout = self.checkout_ref + self.repo._clone_into = self.clone_into + self.assertRaises(morphlib.cachedrepo.CheckoutDirectoryExistsError, self.repo.clone_checkout, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', self.tempfs.root_path) def test_fail_checkout_due_to_clone_error(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._clone_into = self.clone_into + self.assertRaises( morphlib.cachedrepo.CloneError, self.repo.clone_checkout, 'a4da32f5a81c8bc6d660404724cedc3bc0914a75', self.tempfs.getsyspath('failed-checkout')) def test_fail_checkout_due_to_copy_error(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._copy_repository = self.copy_repository + self.assertRaises(morphlib.cachedrepo.CopyError, self.repo.checkout, 'a4da32f5a81c8bc6d660404724cedc3bc0914a75', self.tempfs.getsyspath('failed-checkout')) def test_fail_checkout_from_invalid_ref(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._copy_repository = self.copy_repository + self.repo._checkout_ref_in_clone = self.checkout_ref + self.assertRaises( morphlib.cachedrepo.CheckoutError, self.repo.checkout, '079bbfd447c8534e464ce5d40b80114c2022ebf4', self.tempfs.getsyspath('checkout-from-invalid-ref')) def test_checkout_from_existing_ref_into_new_directory(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._copy_repository = self.copy_repository + self.repo._checkout_ref_in_clone = self.checkout_ref + unpack_dir = self.tempfs.getsyspath('unpack-dir') self.repo.checkout('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', unpack_dir) @@ -219,35 +141,28 @@ class CachedRepoTests(unittest.TestCase): morph_filename = os.path.join(unpack_dir, 'foo.morph') self.assertTrue(os.path.exists(morph_filename)) - def test_ls_tree_in_existing_ref(self): - data = self.repo.ls_tree('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEqual(data, ['foo.morph']) - - def test_fail_ls_tree_in_invalid_ref(self): - self.assertRaises( - morphlib.cachedrepo.InvalidReferenceError, self.repo.ls_tree, - '079bbfd447c8534e464ce5d40b80114c2022ebf4') - - def test_fail_because_ls_tree_in_named_ref_is_not_allowed(self): - self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError, - self.repo.ls_tree, 'master') - def test_successful_update(self): - self.repo._update = self.update_successfully + self.repo._gitdir.update_remotes = self.update_successfully self.repo.update() def test_failing_update(self): - self.repo._update = self.update_with_failure + self.repo._gitdir.update_remotes = self.update_with_failure self.assertRaises(morphlib.cachedrepo.UpdateError, self.repo.update) def test_no_update_if_local(self): - self.repo = morphlib.cachedrepo.CachedRepo( - object(), 'local:repo', 'file:///local/repo/', '/local/repo/') - self.repo._update = self.update_with_failure + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + self.repo = morphlib.cachedrepo.CachedRepo( + object(), 'local:repo', 'file:///local/repo/', '/local/repo/') + self.repo._gitdir.update_remotes = self.update_with_failure + self.repo._gitdir._rev_parse = self.rev_parse + self.assertFalse(self.repo.requires_update_for_ref(self.known_commit)) self.repo.update() def test_clone_checkout(self): + self.repo._gitdir._rev_parse = self.rev_parse + self.repo._clone_into = self.clone_into + self.repo.clone_checkout('master', '/.DOES_NOT_EXIST') self.assertEqual(self.clone_target, '/.DOES_NOT_EXIST') self.assertEqual(self.clone_ref, 'master') @@ -256,11 +171,14 @@ class CachedRepoTests(unittest.TestCase): # If the SHA1 is present locally already there's no need to update. # If it's a named ref then it might have changed in the remote, so we # must still update. + self.repo._gitdir._rev_parse = self.rev_parse + self.assertFalse(self.repo.requires_update_for_ref(self.known_commit)) self.assertTrue(self.repo.requires_update_for_ref('named_ref')) def test_no_need_to_update_repo_if_already_updated(self): - self.repo._update = self.update_successfully + self.repo._gitdir.update_remotes = self.update_successfully + self.repo._gitdir._rev_parse = self.rev_parse self.assertTrue(self.repo.requires_update_for_ref('named_ref')) self.repo.update() diff --git a/morphlib/git.py b/morphlib/git.py index 6a5a9a47..944ddc30 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -275,15 +275,6 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True): gitcmd(runcmd, 'remote', 'update', 'origin', '--prune', cwd=destdir) -def checkout_ref(runcmd, gitdir, ref): - '''Checks out a specific ref/SHA1 in a git working tree.''' - gitcmd(runcmd, 'checkout', ref, cwd=gitdir) - gd = morphlib.gitdir.GitDirectory(gitdir) - if gd.has_fat(): - gd.fat_init() - gd.fat_pull() - - def index_has_changes(runcmd, gitdir): '''Returns True if there are no staged changes to commit''' try: diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index ed71e422..cd395228 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -460,8 +460,7 @@ class GitDirectory(object): def get_blob_contents(self, blob_id): # pragma: no cover '''Get file contents from git by ID''' - return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', - blob_id) + return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', blob_id) def get_commit_contents(self, commit_id): # pragma: no cover '''Get commit contents from git by ID''' @@ -501,15 +500,16 @@ class GitDirectory(object): ''' return Remote(self, *args, **kwargs) - def update_remotes(self): # pragma: no cover + def update_remotes(self, echo_stderr=False): # pragma: no cover '''Run "git remote update --prune".''' - morphlib.git.gitcmd(self._runcmd, 'remote', 'update', '--prune') + morphlib.git.gitcmd(self._runcmd, 'remote', 'update', '--prune', + echo_stderr=echo_stderr) def is_bare(self): '''Determine whether the repository has no work tree (is bare)''' return self.get_config('core.bare') == 'true' - def list_files(self, ref=None): + def list_files(self, ref=None, recurse=True): '''Return an iterable of the files in the repository. If `ref` is specified, list files at that ref, otherwise @@ -522,9 +522,9 @@ class GitDirectory(object): if ref is None and self.is_bare(): raise NoWorkingTreeError(self) if ref is None: - return self._list_files_in_work_tree() + return self._list_files_in_work_tree(recurse) else: - return self._list_files_in_ref(ref) + return self._list_files_in_ref(ref, recurse) def _rev_parse(self, ref): try: @@ -575,31 +575,49 @@ class GitDirectory(object): except InvalidRefError: return False - def _list_files_in_work_tree(self): + def _list_files_in_work_tree(self, recurse=True): for dirpath, subdirs, filenames in os.walk(self.dirname): - if dirpath == self.dirname and '.git' in subdirs: + if not recurse: # pragma: no cover + subdirs[:] = [] + elif dirpath == self.dirname and '.git' in subdirs: subdirs.remove('.git') for filename in filenames: filepath = os.path.join(dirpath, filename) yield os.path.relpath(filepath, start=self.dirname) - def _list_files_in_ref(self, ref): + def _list_files_in_ref(self, ref, recurse=True): tree = self.resolve_ref_to_tree(ref) - output = morphlib.git.gitcmd(self._runcmd, 'ls-tree', - '--name-only', '-rz', tree) + + command = ['ls-tree', '--name-only', '-z'] + if recurse: + command.append('-r') + command.append(tree) + + output = morphlib.git.gitcmd(self._runcmd, *command) # ls-tree appends \0 instead of interspersing, so we need to # strip the trailing \0 before splitting paths = output.strip('\0').split('\0') return paths def read_file(self, filename, ref=None): + '''Attempts to read a file, from the working tree or a given ref. + + Raises an InvalidRefError if the ref is not found in the repository. + Raises an IOError if the requested file is not found in the ref. + + ''' + if ref is None and self.is_bare(): raise NoWorkingTreeError(self) if ref is None: with open(os.path.join(self.dirname, filename)) as f: return f.read() tree = self.resolve_ref_to_tree(ref) - return self.get_file_from_ref(tree, filename) + try: + return self.get_file_from_ref(tree, filename) + except cliapp.AppException: + raise IOError('File %s does not exist in ref %s of repo %s' % + (filename, ref, self)) def is_symlink(self, filename, ref=None): if ref is None and self.is_bare(): diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index fe71ab3a..50de18d7 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -16,6 +16,7 @@ # =*= License: GPL-2 =*= +import contextlib import datetime import os import shutil @@ -25,6 +26,26 @@ import unittest import morphlib +@contextlib.contextmanager +def monkeypatch(obj, attr, new_value): + old_value = getattr(obj, attr) + setattr(obj, attr, new_value) + yield + setattr(obj, attr, old_value) + + +def allow_nonexistant_git_repos(): + '''Disable the gitdir._ensure_is_git_repo() function. + + This is used in other unit tests to avoid needing to run 'git init' at the + start of each test. A library like 'mock' would be a better solution for + this problem. + + ''' + return monkeypatch( + morphlib.gitdir.GitDirectory, '_ensure_is_git_repo', lambda x: None) + + class GitDirectoryTests(unittest.TestCase): def setUp(self): @@ -154,6 +175,12 @@ class GitDirectoryContentsTests(unittest.TestCase): self.assertRaises(morphlib.gitdir.InvalidRefError, gd.read_file, 'bar', 'no-such-ref') + def test_read_raises_io_error(self): + for gitdir in (self.dirname, self.mirror): + gd = morphlib.gitdir.GitDirectory(gitdir) + self.assertRaises(IOError, + gd.read_file, 'non-existant-file', 'HEAD') + def test_HEAD(self): gd = morphlib.gitdir.GitDirectory(self.dirname) self.assertEqual(gd.HEAD, 'master') diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 92c5e763..9bccb20b 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -225,6 +225,11 @@ class LocalRepoCache(object): self.fs.rename(target, path) return self.get_repo(reponame) + def _new_cached_repo_instance(self, reponame, repourl, + path): # pragma: no cover + return morphlib.cachedrepo.CachedRepo( + self._app, reponame, repourl, path) + def get_repo(self, reponame): '''Return an object representing a cached repository.''' @@ -234,8 +239,7 @@ class LocalRepoCache(object): repourl = self._resolver.pull_url(reponame) path = self._cache_name(repourl) if self.fs.exists(path): - repo = morphlib.cachedrepo.CachedRepo(self._app, reponame, - repourl, path) + repo = self._new_cached_repo_instance(reponame, repourl, path) self._cached_repo_objects[reponame] = repo return repo raise NotCached(reponame) diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index 3cc4f07f..ab6e71fd 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -22,6 +22,7 @@ import cliapp import fs.memoryfs import morphlib +import morphlib.gitdir_tests class FakeApplication(object): @@ -56,6 +57,7 @@ class LocalRepoCacheTests(unittest.TestCase): self.lrc._git = self.fake_git self.lrc._fetch = self.not_found self.lrc._mkdtemp = self.fake_mkdtemp + self.lrc._new_cached_repo_instance = self.new_cached_repo_instance self._mkdtemp_count = 0 def fake_git(self, args, **kwargs): @@ -86,6 +88,11 @@ class LocalRepoCacheTests(unittest.TestCase): self.lrc.fs.makedir(dirname+"/"+thing) return thing + def new_cached_repo_instance(self, *args): + with morphlib.gitdir_tests.allow_nonexistant_git_repos(): + return morphlib.cachedrepo.CachedRepo( + FakeApplication(), *args) + def not_found(self, url, path): raise cliapp.AppException('Not found') diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index dad7238e..a3ac2749 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -57,10 +57,11 @@ class MorphologyFactory(object): chatty=True) try: repo = self._lrc.get_repo(reponame) - morph = loader.load_from_string(repo.cat(sha1, filename)) + text = repo.read_file(filename, sha1) + morph = loader.load_from_string(text) except IOError: morph = None - file_list = repo.ls_tree(sha1) + file_list = repo.list_files(ref=sha1, recurse=False) elif self._rrc is not None: self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" " from the remote git cache.", diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 41d06480..5222ca6d 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -37,6 +37,7 @@ class FakeRemoteRepoCache(object): def ls_tree(self, reponame, sha1): return [] + class FakeLocalRepo(object): morphologies = { @@ -115,7 +116,7 @@ class FakeLocalRepo(object): def __init__(self): self.arch = 'x86_64' - def cat(self, sha1, filename): + def read_file(self, filename, ref): if filename in self.morphologies: values = { 'arch': self.arch, @@ -129,9 +130,10 @@ class FakeLocalRepo(object): }''' % filename[:-len('.morph')] return 'text' - def ls_tree(self, sha1): + def list_files(self, ref, recurse): return self.morphologies.keys() + class FakeLocalRepoCache(object): def __init__(self, lr): @@ -163,14 +165,14 @@ class MorphologyFactoryTests(unittest.TestCase): return ['chunk.morph'] def nolocalmorph(self, *args): - if args[-1].endswith('.morph'): + if args[0].endswith('.morph'): raise IOError('File not found') return 'text' - def autotoolsbuildsystem(self, *args): + def autotoolsbuildsystem(self, *args, **kwargs): return ['configure.in'] - def remotemorph(self, *args): + def remotemorph(self, *args, **kwargs): return ['remote-chunk.morph'] def noremotemorph(self, *args): @@ -182,7 +184,7 @@ class MorphologyFactoryTests(unittest.TestCase): return False def test_gets_morph_from_local_repo(self): - self.lr.ls_tree = self.localmorph + self.lr.list_files = self.localmorph morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual('chunk', morph['name']) @@ -195,8 +197,8 @@ class MorphologyFactoryTests(unittest.TestCase): self.assertEqual('remote-chunk', morph['name']) def test_autodetects_local_morphology(self): - self.lr.cat = self.nolocalmorph - self.lr.ls_tree = self.autotoolsbuildsystem + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem morph = self.mf.get_morphology('reponame', 'sha1', 'assumed-local.morph') self.assertEqual('assumed-local', morph['name']) @@ -210,7 +212,7 @@ class MorphologyFactoryTests(unittest.TestCase): self.assertEqual('assumed-remote', morph['name']) def test_raises_error_when_no_local_morph(self): - self.lr.cat = self.nolocalfile + self.lr.read_file = self.nolocalfile self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology, 'reponame', 'sha1', 'unreached.morph') @@ -225,14 +227,14 @@ class MorphologyFactoryTests(unittest.TestCase): 'reponame', 'sha1', 'name-mismatch.morph') def test_looks_locally_with_no_remote(self): - self.lr.ls_tree = self.localmorph + self.lr.list_files = self.localmorph morph = self.lmf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual('chunk', morph['name']) def test_autodetects_locally_with_no_remote(self): - self.lr.cat = self.nolocalmorph - self.lr.ls_tree = self.autotoolsbuildsystem + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem morph = self.mf.get_morphology('reponame', 'sha1', 'assumed-local.morph') self.assertEqual('assumed-local', morph['name']) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 74645f41..2d1fe979 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -51,7 +51,7 @@ class ProjectVersionGuesser(object): if self.lrc.has_repo(repo): repository = self.lrc.get_repo(repo) for filename in filenames: - yield filename, repository.cat(ref, filename) + yield filename, repository.read_file(filename, ref) elif self.rrc: for filename in filenames: yield filename, self.rrc.cat_file(repo, ref, filename) @@ -153,7 +153,7 @@ class VersionGuesser(object): repository = self.lrc.get_repo(repo) if not self.app.settings['no-git-update']: repository.update() - tree = repository.ls_tree(ref) + tree = repository.list_files(ref=ref, recurse=False) elif self.rrc: repository = None tree = self.rrc.ls_tree(repo, ref) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 5531f7f6..76da9736 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -174,7 +174,7 @@ class BranchAndMergePlugin(cliapp.Plugin): cached_repo = lrc.get_updated_repo(root_url) # Check the git branch exists. - cached_repo.resolve_ref(system_branch) + cached_repo.resolve_ref_to_commit(system_branch) with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): @@ -233,7 +233,7 @@ class BranchAndMergePlugin(cliapp.Plugin): (system_branch, root_url)) # Make sure the base_ref exists. - cached_repo.resolve_ref(base_ref) + cached_repo.resolve_ref_to_commit(base_ref) with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 8daaba81..3a328eb7 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -71,7 +71,8 @@ class SourceResolver(object): repo.update() # If the user passed --no-git-update, and the ref is a SHA1 not # available locally, this call will raise an exception. - absref, tree = repo.resolve_ref(ref) + absref = repo.resolve_ref_to_commit(ref) + tree = repo.resolve_ref_to_tree(absref) elif self.rrc is not None: try: absref, tree = self.rrc.resolve_ref(reponame, ref) @@ -91,7 +92,8 @@ class SourceResolver(object): repo.update() else: repo = self.lrc.get_repo(reponame) - absref, tree = repo.resolve_ref(ref) + absref = repo.resolve_ref_to_commit(ref) + tree = repo.resolve_ref_to_tree(absref) return absref, tree def traverse_morphs(self, definitions_repo, definitions_ref, diff --git a/tests.branching/branch-cleans-up-on-failure.stderr b/tests.branching/branch-cleans-up-on-failure.stderr index 37533408..959226d8 100644 --- a/tests.branching/branch-cleans-up-on-failure.stderr +++ b/tests.branching/branch-cleans-up-on-failure.stderr @@ -1 +1 @@ -ERROR: Ref invalid-ref is an invalid reference for repo file://TMP/morphs +ERROR: Git directory TMP/morphs has no commit at ref invalid-ref^{commit}. diff --git a/tests.branching/checkout-cleans-up-on-failure.stderr b/tests.branching/checkout-cleans-up-on-failure.stderr index 5b6a5645..14ed130c 100644 --- a/tests.branching/checkout-cleans-up-on-failure.stderr +++ b/tests.branching/checkout-cleans-up-on-failure.stderr @@ -1 +1 @@ -ERROR: Ref i/do/not/exist is an invalid reference for repo file://TMP/morphs +ERROR: Git directory TMP/morphs has no commit at ref i/do/not/exist^{commit}. diff --git a/tests.build/missing-ref.stderr b/tests.build/missing-ref.stderr index 5fa5456b..b5139e25 100644 --- a/tests.build/missing-ref.stderr +++ b/tests.build/missing-ref.stderr @@ -1 +1 @@ -ERROR: Ref non-existent-branch is an invalid reference for repo file://TMP/morphs-repo +ERROR: Git directory TMP/morphs-repo has no commit at ref non-existent-branch^{commit}. |