Merge branch 'sam/cached-repo-cleanup'

Reviewed-By: Adam Coldrick <adam.coldrick@codethink.co.uk>
Reviewed-By: Richard Maw <richard.maw@codethink.co.uk>

17 files changed, 214 insertions, 318 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 5c7d5c5e..a22e689b 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -399,13 +399,13 @@ class BuildCommand(object):
             source.repo = self.lrc.get_repo(repo_name)
             try:
                 sha1 = source.sha1
-                source.repo.resolve_ref(sha1)
+                source.repo.resolve_ref_to_commit(sha1)
                 self.app.status(msg='Not updating git repository '
                                     '%(repo_name)s because it '
                                     'already contains sha1 %(sha1)s',
                                 chatty=True, repo_name=repo_name,
                                 sha1=sha1)
-            except morphlib.cachedrepo.InvalidReferenceError:
+            except morphlib.gitdir.InvalidRefError:
                 self.app.status(msg='Updating %(repo_name)s',
                                 repo_name=repo_name)
                 source.repo.update()
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 2087c942..245648ba 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -20,6 +20,7 @@ import StringIO
 import unittest
 
 import morphlib
+import morphlib.gitdir_tests
 
 
 class FakeBuildSystem(object):
@@ -50,8 +51,9 @@ class FakeSource(object):
         }
         self.name = 'a'
 
-        self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo',
-                                                   'url', 'path')
+        with morphlib.gitdir_tests.allow_nonexistant_git_repos():
+            self.repo = morphlib.cachedrepo.CachedRepo(
+                FakeApp(), 'repo', 'url', 'path')
         self.repo_name = 'url'
         self.original_ref = 'e'
         self.sha1 = 'f'
diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py
index aad3d84e..aa2b5af1 100644
--- a/morphlib/cachedrepo.py
+++ b/morphlib/cachedrepo.py
@@ -15,26 +15,11 @@
 
 
 import cliapp
-import logging
 import os
 
 import morphlib
 
 
-class InvalidReferenceError(cliapp.AppException):
-
-    def __init__(self, repo, ref):
-        cliapp.AppException.__init__(
-            self, 'Ref %s is an invalid reference for repo %s' % (ref, repo))
-
-
-class UnresolvedNamedReferenceError(cliapp.AppException):
-
-    def __init__(self, repo, ref):
-        cliapp.AppException.__init__(
-            self, 'Ref %s is not a SHA1 ref for repo %s' % (ref, repo))
-
-
 class CheckoutDirectoryExistsError(cliapp.AppException):
 
     def __init__(self, repo, target_dir):
@@ -105,63 +90,52 @@ class CachedRepo(object):
         self.is_mirror = not url.startswith('file://')
         self.already_updated = False
 
-    def ref_exists(self, ref):
-        '''Returns True if the given ref exists in the repo'''
+        self._gitdir = morphlib.gitdir.GitDirectory(path)
 
-        try:
-            self._rev_parse(ref)
-        except cliapp.AppException:
-            return False
-        return True
+    def ref_exists(self, ref):  # pragma: no cover
+        '''Returns True if the given ref exists in the repo'''
+        return self._gitdir.ref_exists(ref)
 
-    def resolve_ref(self, ref):
-        '''Attempts to resolve a ref into its SHA1 and tree SHA1.
+    def resolve_ref_to_commit(self, ref):  # pragma: no cover
+        '''Resolve a named ref to a commit SHA1.
 
-        Raises an InvalidReferenceError if the ref is not found in the
-        repository.
+        Raises gitdir.InvalidRefError if the ref does not exist.
 
         '''
+        return self._gitdir.resolve_ref_to_commit(ref)
 
-        try:
-            absref = self._rev_parse(ref)
-        except cliapp.AppException:
-            raise InvalidReferenceError(self, ref)
+    def resolve_ref_to_tree(self, ref):  # pragma: no cover
+        '''Resolve a named ref to a tree SHA1.
 
-        try:
-            tree = self._show_tree_hash(absref)
-        except cliapp.AppException:
-            raise InvalidReferenceError(self, ref)
+        Raises gitdir.InvalidRefError if the ref does not exist.
 
-        return absref, tree
+        '''
+        return self._gitdir.resolve_ref_to_tree(ref)
 
-    def cat(self, ref, filename):
-        '''Attempts to read a file given a SHA1 ref.
+    def read_file(self, filename, ref):  # pragma: no cover
+        '''Attempts to read a file from a given ref.
 
-        Raises an UnresolvedNamedReferenceError if the ref is not a SHA1
-        ref. Raises an InvalidReferenceError if the SHA1 ref is not found
-        in the repository. Raises an IOError if the requested file is not
-        found in the ref.
+        Raises a gitdir.InvalidRefError if the ref is not found in the
+        repository. Raises an IOError if the requested file is not found in
+        the ref.
 
         '''
+        return self._gitdir.read_file(filename, ref)
 
-        if not morphlib.git.is_valid_sha1(ref):
-            raise UnresolvedNamedReferenceError(self, ref)
-        try:
-            sha1 = self._rev_parse(ref)
-        except cliapp.AppException:
-            raise InvalidReferenceError(self, ref)
+    def list_files(self, ref, recurse=True):  # pragma: no cover
+        '''Return filenames found in the tree pointed to by the given ref.
 
-        try:
-            return self._cat_file(sha1, filename)
-        except cliapp.AppException:
-            raise IOError('File %s does not exist in ref %s of repo %s' %
-                          (filename, ref, self))
+        Returns a gitdir.InvalidRefError if the ref is not found in the
+        repository.
+
+        '''
+        return self._gitdir.list_files(ref, recurse)
 
     def clone_checkout(self, ref, target_dir):
         '''Clone from the cache into the target path and check out a given ref.
 
         Raises a CheckoutDirectoryExistsError if the target
-        directory already exists. Raises an InvalidReferenceError if the
+        directory already exists. Raises a gitdir.InvalidRefError if the
         ref is not found in the repository. Raises a CheckoutError if
         something else goes wrong while copying the repository or checking
         out the SHA1 ref.
@@ -171,14 +145,14 @@ class CachedRepo(object):
         if os.path.exists(target_dir):
             raise CheckoutDirectoryExistsError(self, target_dir)
 
-        self.resolve_ref(ref)
+        self._gitdir.resolve_ref_to_commit(ref)
 
         self._clone_into(target_dir, ref)
 
     def checkout(self, ref, target_dir):
         '''Unpacks the repository in a directory and checks out a commit ref.
 
-        Raises an InvalidReferenceError if the ref is not found in the
+        Raises an gitdir.InvalidRefError if the ref is not found in the
         repository. Raises a CopyError if something goes wrong with the copy
         of the repository. Raises a CheckoutError if something else goes wrong
         while copying the repository or checking out the SHA1 ref.
@@ -193,25 +167,7 @@ class CachedRepo(object):
         # take care to turn the copy into something as good as a real clone.
         self._copy_repository(self.path, target_dir)
 
-        self._checkout_ref(ref, target_dir)
-
-    def ls_tree(self, ref):
-        '''Return file names found in root tree. Does not recurse to subtrees.
-
-        Raises an UnresolvedNamedReferenceError if the ref is not a SHA1
-        ref. Raises an InvalidReferenceError if the SHA1 ref is not found
-        in the repository.
-
-        '''
-
-        if not morphlib.git.is_valid_sha1(ref):
-            raise UnresolvedNamedReferenceError(self, ref)
-        try:
-            sha1 = self._rev_parse(ref)
-        except cliapp.AppException:
-            raise InvalidReferenceError(self, ref)
-
-        return self._ls_tree(sha1)
+        self._checkout_ref_in_clone(ref, target_dir)
 
     def requires_update_for_ref(self, ref):
         '''Returns False if there's no need to update this cached repo.
@@ -232,7 +188,7 @@ class CachedRepo(object):
         # Named refs that are valid SHA1s will confuse this code.
         ref_can_change = not morphlib.git.is_valid_sha1(ref)
 
-        if ref_can_change or not self.ref_exists(ref):
+        if ref_can_change or not self._gitdir.ref_exists(ref):
             return True
         else:
             return False
@@ -249,9 +205,10 @@ class CachedRepo(object):
             return
 
         try:
-            self._update()
+            self._gitdir.update_remotes(
+                echo_stderr=self.app.settings['verbose'])
             self.already_updated = True
-        except cliapp.AppException, e:
+        except cliapp.AppException:
             raise UpdateError(self)
 
     def _runcmd(self, *args, **kwargs):  # pragma: no cover
@@ -259,27 +216,11 @@ class CachedRepo(object):
             kwargs['cwd'] = self.path
         return self.app.runcmd(*args, **kwargs)
 
-    def _rev_parse(self, ref):  # pragma: no cover
-        return morphlib.git.gitcmd(self._runcmd, 'rev-parse', '--verify',
-                                   '%s^{commit}' % ref)[0:40]
-
-    def _show_tree_hash(self, absref):  # pragma: no cover
-        return morphlib.git.gitcmd(self._runcmd, 'rev-parse', '--verify',
-                                   '%s^{tree}' % absref).strip()
-
-    def _ls_tree(self, ref):  # pragma: no cover
-        result = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
-                                     '--name-only', ref)
-        return result.split('\n')
-
-    def _cat_file(self, ref, filename):  # pragma: no cover
-        return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob',
-                                   '%s:%s' % (ref, filename))
-
-    def _clone_into(self, target_dir, ref):  #pragma: no cover
+    def _clone_into(self, target_dir, ref):  # pragma: no cover
         '''Actually perform the clone'''
         try:
-            morphlib.git.clone_into(self._runcmd, self.path, target_dir, ref)
+            morphlib.git.clone_into(self._runcmd, self.path, target_dir,
+                                    ref)
         except cliapp.AppException:
             raise CloneError(self, target_dir)
 
@@ -290,16 +231,15 @@ class CachedRepo(object):
         except cliapp.AppException:
             raise CopyError(self, target_dir)
 
-    def _checkout_ref(self, ref, target_dir):  # pragma: no cover
+    def _checkout_ref_in_clone(self, ref, clone_dir):  # pragma: no cover
+        # This is a separate GitDirectory instance. Don't confuse it with the
+        # internal ._gitdir attribute!
+        working_gitdir = morphlib.gitdir.GitDirectory(clone_dir)
         try:
-            morphlib.git.checkout_ref(self._runcmd, target_dir, ref)
-        except cliapp.AppException:
-            raise CheckoutError(self, ref, target_dir)
-
-    def _update(self):  # pragma: no cover
-        morphlib.git.gitcmd(self._runcmd, 'remote', 'update',
-                            'origin', '--prune',
-                            echo_stderr=self.app.settings['verbose'])
+            working_gitdir.checkout(ref)
+        except cliapp.AppException as e:
+            raise CheckoutError(self, ref, clone_dir)
+        return working_gitdir
 
     def __str__(self):  # pragma: no cover
         return self.url
diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py
index d3ae331a..6f87bfdd 100644
--- a/morphlib/cachedrepo_tests.py
+++ b/morphlib/cachedrepo_tests.py
@@ -22,14 +22,18 @@ import fs.tempfs
 import cliapp
 
 import morphlib
+import morphlib.gitdir_tests
 
 
-class CachedRepoTests(unittest.TestCase):
+class FakeApplication(object):
+
+    def __init__(self):
+        self.settings = {
+            'verbose': True
+        }
+
 
-    EXAMPLE_MORPH = '''{
-        "name": "foo",
-        "kind": "chunk"
-    }'''
+class CachedRepoTests(unittest.TestCase):
 
     known_commit = 'a4da32f5a81c8bc6d660404724cedc3bc0914a75'
     bad_sha1_known_to_rev_parse = 'cafecafecafecafecafecafecafecafecafecafe'
@@ -44,85 +48,43 @@ class CachedRepoTests(unittest.TestCase):
             'master': 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
             'baserock/morph': '8b780e2e6f102fcf400ff973396566d36d730501'
         }
+        ref = ref.rstrip('^{commit}')
         try:
             return output[ref]
         except KeyError:
             raise cliapp.AppException('git rev-parse --verify %s' % ref)
 
-    def show_tree_hash(self, absref):
-        output = {
-            'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9':
-              'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
-            '8b780e2e6f102fcf400ff973396566d36d730501':
-              'ffffffffffffffffffffffffffffffffffffffff',
-            'a4da32f5a81c8bc6d660404724cedc3bc0914a75':
-              'dddddddddddddddddddddddddddddddddddddddd'
-        }
-        try:
-            return output[absref]
-        except KeyError:
-            raise cliapp.AppException('git log -1 --format=format:%%T %s' %
-                                      absref)
-
-    def cat_file(self, ref, filename):
-        output = {
-            'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9:foo.morph':
-            self.EXAMPLE_MORPH
-        }
-        try:
-            return output['%s:%s' % (ref, filename)]
-        except KeyError:
-            raise cliapp.AppException(
-                'git cat-file blob %s:%s' % (ref, filename))
-
     def copy_repository(self, source_dir, target_dir):
         if target_dir.endswith('failed-checkout'):
             raise morphlib.cachedrepo.CopyError(self.repo, target_dir)
 
     def checkout_ref(self, ref, target_dir):
-        if ref == 'a4da32f5a81c8bc6d660404724cedc3bc0914a75':
-            raise morphlib.cachedrepo.CloneError(self.repo, target_dir)
-        elif ref == '079bbfd447c8534e464ce5d40b80114c2022ebf4':
+        if ref == '079bbfd447c8534e464ce5d40b80114c2022ebf4':
             raise morphlib.cachedrepo.CheckoutError(self.repo, ref, target_dir)
         else:
             with open(os.path.join(target_dir, 'foo.morph'), 'w') as f:
                 f.write('contents of foo.morph')
 
-    def ls_tree(self, ref):
-        output = {
-            'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9':
-            ['foo.morph']
-        }
-        try:
-            return output[ref]
-        except KeyError:
-            raise cliapp.AppException('git ls-tree --name-only %s' % (ref))
-
     def clone_into(self, target_dir, ref):
         if target_dir.endswith('failed-checkout'):
             raise morphlib.cachedrepo.CloneError(self.repo, target_dir)
         self.clone_target = target_dir
         self.clone_ref = ref
 
-    def update_successfully(self):
+    def update_successfully(self, **kwargs):
         pass
 
-    def update_with_failure(self):
+    def update_with_failure(self, **kwargs):
         raise cliapp.AppException('git remote update origin')
 
     def setUp(self):
         self.repo_name = 'foo'
         self.repo_url = 'git://foo.bar/foo.git'
         self.repo_path = '/tmp/foo'
-        self.repo = morphlib.cachedrepo.CachedRepo(
-            object(), self.repo_name, self.repo_url, self.repo_path)
-        self.repo._rev_parse = self.rev_parse
-        self.repo._show_tree_hash = self.show_tree_hash
-        self.repo._cat_file = self.cat_file
-        self.repo._copy_repository = self.copy_repository
-        self.repo._checkout_ref = self.checkout_ref
-        self.repo._ls_tree = self.ls_tree
-        self.repo._clone_into = self.clone_into
+        with morphlib.gitdir_tests.allow_nonexistant_git_repos():
+            self.repo = morphlib.cachedrepo.CachedRepo(
+                FakeApplication(), self.repo_name, self.repo_url,
+                self.repo_path)
         self.tempfs = fs.tempfs.TempFS()
 
     def test_constructor_sets_name_and_url_and_path(self):
@@ -130,87 +92,47 @@ class CachedRepoTests(unittest.TestCase):
         self.assertEqual(self.repo.url, self.repo_url)
         self.assertEqual(self.repo.path, self.repo_path)
 
-    def test_ref_exists(self):
-        self.assertEqual(self.repo.ref_exists('master'), True)
-
-    def test_ref_does_not_exist(self):
-        self.assertEqual(self.repo.ref_exists('non-existant-ref'), False)
-
-    def test_resolve_named_ref_master(self):
-        sha1, tree = self.repo.resolve_ref('master')
-        self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee')
-
-    def test_resolve_named_ref_baserock_morph(self):
-        sha1, tree = self.repo.resolve_ref('baserock/morph')
-        self.assertEqual(sha1, '8b780e2e6f102fcf400ff973396566d36d730501')
-        self.assertEqual(tree, 'ffffffffffffffffffffffffffffffffffffffff')
-
-    def test_fail_resolving_invalid_named_ref(self):
-        self.assertRaises(morphlib.cachedrepo.InvalidReferenceError,
-                          self.repo.resolve_ref, 'foo/bar')
-
-    def test_resolve_sha1_ref(self):
-        sha1, tree = self.repo.resolve_ref(
-            'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee')
-
-    def test_fail_resolving_an_invalid_sha1_ref(self):
-        self.assertRaises(morphlib.cachedrepo.InvalidReferenceError,
-                          self.repo.resolve_ref,
-                          self.bad_sha1_known_to_rev_parse)
-
-    def test_cat_existing_file_in_existing_ref(self):
-        data = self.repo.cat('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
-                             'foo.morph')
-        self.assertEqual(data, self.EXAMPLE_MORPH)
-
-    def test_fail_cat_file_in_invalid_ref(self):
-        self.assertRaises(
-            morphlib.cachedrepo.InvalidReferenceError, self.repo.cat,
-            '079bbfd447c8534e464ce5d40b80114c2022ebf4',
-            'doesnt-matter-whether-this-file-exists')
-
-    def test_fail_cat_non_existent_file_in_existing_ref(self):
-        self.assertRaises(IOError, self.repo.cat,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
-                          'file-that-does-not-exist')
-
-    def test_fail_cat_non_existent_file_in_invalid_ref(self):
-        self.assertRaises(
-            morphlib.cachedrepo.InvalidReferenceError, self.repo.cat,
-            '079bbfd447c8534e464ce5d40b80114c2022ebf4',
-            'file-that-does-not-exist')
-
-    def test_fail_because_cat_in_named_ref_is_not_allowed(self):
-        self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError,
-                          self.repo.cat, 'master', 'doesnt-matter')
-
     def test_fail_clone_checkout_into_existing_directory(self):
+        self.repo._gitdir.checkout = self.checkout_ref
+        self.repo._clone_into = self.clone_into
+
         self.assertRaises(morphlib.cachedrepo.CheckoutDirectoryExistsError,
                           self.repo.clone_checkout,
                           'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
                           self.tempfs.root_path)
 
     def test_fail_checkout_due_to_clone_error(self):
+        self.repo._gitdir._rev_parse = self.rev_parse
+        self.repo._clone_into = self.clone_into
+
         self.assertRaises(
             morphlib.cachedrepo.CloneError, self.repo.clone_checkout,
             'a4da32f5a81c8bc6d660404724cedc3bc0914a75',
             self.tempfs.getsyspath('failed-checkout'))
 
     def test_fail_checkout_due_to_copy_error(self):
+        self.repo._gitdir._rev_parse = self.rev_parse
+        self.repo._copy_repository = self.copy_repository
+
         self.assertRaises(morphlib.cachedrepo.CopyError, self.repo.checkout,
                           'a4da32f5a81c8bc6d660404724cedc3bc0914a75',
                           self.tempfs.getsyspath('failed-checkout'))
 
     def test_fail_checkout_from_invalid_ref(self):
+        self.repo._gitdir._rev_parse = self.rev_parse
+        self.repo._copy_repository = self.copy_repository
+        self.repo._checkout_ref_in_clone = self.checkout_ref
+
         self.assertRaises(
             morphlib.cachedrepo.CheckoutError, self.repo.checkout,
             '079bbfd447c8534e464ce5d40b80114c2022ebf4',
             self.tempfs.getsyspath('checkout-from-invalid-ref'))
 
     def test_checkout_from_existing_ref_into_new_directory(self):
+        self.repo._gitdir._rev_parse = self.rev_parse
+        self.repo._copy_repository = self.copy_repository
+        self.repo._checkout_ref_in_clone = self.checkout_ref
+
         unpack_dir = self.tempfs.getsyspath('unpack-dir')
         self.repo.checkout('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
                            unpack_dir)
@@ -219,35 +141,28 @@ class CachedRepoTests(unittest.TestCase):
         morph_filename = os.path.join(unpack_dir, 'foo.morph')
         self.assertTrue(os.path.exists(morph_filename))
 
-    def test_ls_tree_in_existing_ref(self):
-        data = self.repo.ls_tree('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEqual(data, ['foo.morph'])
-
-    def test_fail_ls_tree_in_invalid_ref(self):
-        self.assertRaises(
-            morphlib.cachedrepo.InvalidReferenceError, self.repo.ls_tree,
-            '079bbfd447c8534e464ce5d40b80114c2022ebf4')
-
-    def test_fail_because_ls_tree_in_named_ref_is_not_allowed(self):
-        self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError,
-                          self.repo.ls_tree, 'master')
-
     def test_successful_update(self):
-        self.repo._update = self.update_successfully
+        self.repo._gitdir.update_remotes = self.update_successfully
         self.repo.update()
 
     def test_failing_update(self):
-        self.repo._update = self.update_with_failure
+        self.repo._gitdir.update_remotes = self.update_with_failure
         self.assertRaises(morphlib.cachedrepo.UpdateError, self.repo.update)
 
     def test_no_update_if_local(self):
-        self.repo = morphlib.cachedrepo.CachedRepo(
-            object(), 'local:repo', 'file:///local/repo/', '/local/repo/')
-        self.repo._update = self.update_with_failure
+        with morphlib.gitdir_tests.allow_nonexistant_git_repos():
+            self.repo = morphlib.cachedrepo.CachedRepo(
+                object(), 'local:repo', 'file:///local/repo/', '/local/repo/')
+        self.repo._gitdir.update_remotes = self.update_with_failure
+        self.repo._gitdir._rev_parse = self.rev_parse
+
         self.assertFalse(self.repo.requires_update_for_ref(self.known_commit))
         self.repo.update()
 
     def test_clone_checkout(self):
+        self.repo._gitdir._rev_parse = self.rev_parse
+        self.repo._clone_into = self.clone_into
+
         self.repo.clone_checkout('master', '/.DOES_NOT_EXIST')
         self.assertEqual(self.clone_target, '/.DOES_NOT_EXIST')
         self.assertEqual(self.clone_ref, 'master')
@@ -256,11 +171,14 @@ class CachedRepoTests(unittest.TestCase):
         # If the SHA1 is present locally already there's no need to update.
         # If it's a named ref then it might have changed in the remote, so we
         # must still update.
+        self.repo._gitdir._rev_parse = self.rev_parse
+
         self.assertFalse(self.repo.requires_update_for_ref(self.known_commit))
         self.assertTrue(self.repo.requires_update_for_ref('named_ref'))
 
     def test_no_need_to_update_repo_if_already_updated(self):
-        self.repo._update = self.update_successfully
+        self.repo._gitdir.update_remotes = self.update_successfully
+        self.repo._gitdir._rev_parse = self.rev_parse
 
         self.assertTrue(self.repo.requires_update_for_ref('named_ref'))
         self.repo.update()
diff --git a/morphlib/git.py b/morphlib/git.py
index 6a5a9a47..456aa4c0 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -14,7 +14,6 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import binascii
 import cliapp
 import ConfigParser
 import logging
@@ -23,10 +22,6 @@ import re
 import string
 import StringIO
 import sys
-import time
-
-
-import cliapp
 
 import morphlib
 
@@ -226,11 +221,6 @@ def check_config_set(runcmd, keys, cwd='.'):
     return found
 
 
-def set_remote(runcmd, gitdir, name, url):
-    '''Set remote with name 'name' use a given url at gitdir'''
-    return gitcmd(runcmd, 'remote', 'set-url', name, url, cwd=gitdir)
-
-
 def copy_repository(runcmd, repo, destdir, is_mirror=True):
     '''Copies a cached repository into a directory using cp.
 
@@ -275,25 +265,6 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True):
     gitcmd(runcmd, 'remote', 'update', 'origin', '--prune', cwd=destdir)
 
 
-def checkout_ref(runcmd, gitdir, ref):
-    '''Checks out a specific ref/SHA1 in a git working tree.'''
-    gitcmd(runcmd, 'checkout', ref, cwd=gitdir)
-    gd = morphlib.gitdir.GitDirectory(gitdir)
-    if gd.has_fat():
-        gd.fat_init()
-        gd.fat_pull()
-
-
-def index_has_changes(runcmd, gitdir):
-    '''Returns True if there are no staged changes to commit'''
-    try:
-        gitcmd(runcmd, 'diff-index', '--cached', '--quiet',
-               '--ignore-submodules', 'HEAD', cwd=gitdir)
-    except cliapp.AppException:
-        return True
-    return False
-
-
 def reset_workdir(runcmd, gitdir):
     '''Removes any differences between the current commit '''
     '''and the status of the working directory'''
@@ -321,10 +292,6 @@ def is_valid_sha1(ref):
 
     return len(ref) == 40 and all(x in string.hexdigits for x in ref)
 
-def rev_parse(runcmd, gitdir, ref):
-    '''Find the sha1 for the given ref'''
-    return gitcmd(runcmd, 'rev-parse', '--verify', ref, cwd=gitdir)[0:40]
-
 
 def gitcmd(runcmd, *args, **kwargs):
     '''Run git commands safely'''
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 4c3efbd3..74db6ed6 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -460,8 +460,7 @@ class GitDirectory(object):
 
     def get_blob_contents(self, blob_id): # pragma: no cover
         '''Get file contents from git by ID'''
-        return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob',
-                                   blob_id)
+        return morphlib.git.gitcmd(self._runcmd, 'cat-file', 'blob', blob_id)
 
     def get_commit_contents(self, commit_id): # pragma: no cover
         '''Get commit contents from git by ID'''
@@ -505,15 +504,16 @@ class GitDirectory(object):
         '''
         return Remote(self, *args, **kwargs)
 
-    def update_remotes(self): # pragma: no cover
+    def update_remotes(self, echo_stderr=False): # pragma: no cover
         '''Run "git remote update --prune".'''
-        morphlib.git.gitcmd(self._runcmd, 'remote', 'update', '--prune')
+        morphlib.git.gitcmd(self._runcmd, 'remote', 'update', '--prune',
+                            echo_stderr=echo_stderr)
 
     def is_bare(self):
         '''Determine whether the repository has no work tree (is bare)'''
         return self.get_config('core.bare') == 'true'
 
-    def list_files(self, ref=None):
+    def list_files(self, ref=None, recurse=True):
         '''Return an iterable of the files in the repository.
 
         If `ref` is specified, list files at that ref, otherwise
@@ -526,9 +526,9 @@ class GitDirectory(object):
         if ref is None and self.is_bare():
             raise NoWorkingTreeError(self)
         if ref is None:
-            return self._list_files_in_work_tree()
+            return self._list_files_in_work_tree(recurse)
         else:
-            return self._list_files_in_ref(ref)
+            return self._list_files_in_ref(ref, recurse)
 
     def _rev_parse(self, ref):
         try:
@@ -579,31 +579,49 @@ class GitDirectory(object):
         except InvalidRefError:
             return False
 
-    def _list_files_in_work_tree(self):
+    def _list_files_in_work_tree(self, recurse=True):
         for dirpath, subdirs, filenames in os.walk(self.dirname):
-            if dirpath == self.dirname and '.git' in subdirs:
+            if not recurse:  # pragma: no cover
+                subdirs[:] = []
+            elif dirpath == self.dirname and '.git' in subdirs:
                 subdirs.remove('.git')
             for filename in filenames:
                 filepath = os.path.join(dirpath, filename)
                 yield os.path.relpath(filepath, start=self.dirname)
 
-    def _list_files_in_ref(self, ref):
+    def _list_files_in_ref(self, ref, recurse=True):
         tree = self.resolve_ref_to_tree(ref)
-        output = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
-                                     '--name-only', '-rz', tree)
+
+        command = ['ls-tree', '--name-only', '-z']
+        if recurse:
+            command.append('-r')
+        command.append(tree)
+
+        output = morphlib.git.gitcmd(self._runcmd, *command)
         # ls-tree appends \0 instead of interspersing, so we need to
         # strip the trailing \0 before splitting
         paths = output.strip('\0').split('\0')
         return paths
 
     def read_file(self, filename, ref=None):
+        '''Attempts to read a file, from the working tree or a given ref.
+
+        Raises an InvalidRefError if the ref is not found in the repository.
+        Raises an IOError if the requested file is not found in the ref.
+
+        '''
+
         if ref is None and self.is_bare():
             raise NoWorkingTreeError(self)
         if ref is None:
             with open(os.path.join(self.dirname, filename)) as f:
                 return f.read()
         tree = self.resolve_ref_to_tree(ref)
-        return self.get_file_from_ref(tree, filename)
+        try:
+            return self.get_file_from_ref(tree, filename)
+        except cliapp.AppException:
+            raise IOError('File %s does not exist in ref %s of repo %s' %
+                          (filename, ref, self))
 
     def is_symlink(self, filename, ref=None):
         if ref is None and self.is_bare():
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index fe71ab3a..50de18d7 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -16,6 +16,7 @@
 # =*= License: GPL-2 =*=
 
 
+import contextlib
 import datetime
 import os
 import shutil
@@ -25,6 +26,26 @@ import unittest
 import morphlib
 
 
+@contextlib.contextmanager
+def monkeypatch(obj, attr, new_value):
+    old_value = getattr(obj, attr)
+    setattr(obj, attr, new_value)
+    yield
+    setattr(obj, attr, old_value)
+
+
+def allow_nonexistant_git_repos():
+    '''Disable the gitdir._ensure_is_git_repo() function.
+
+    This is used in other unit tests to avoid needing to run 'git init' at the
+    start of each test. A library like 'mock' would be a better solution for
+    this problem.
+
+    '''
+    return monkeypatch(
+        morphlib.gitdir.GitDirectory, '_ensure_is_git_repo', lambda x: None)
+
+
 class GitDirectoryTests(unittest.TestCase):
 
     def setUp(self):
@@ -154,6 +175,12 @@ class GitDirectoryContentsTests(unittest.TestCase):
         self.assertRaises(morphlib.gitdir.InvalidRefError,
                           gd.read_file, 'bar', 'no-such-ref')
 
+    def test_read_raises_io_error(self):
+        for gitdir in (self.dirname, self.mirror):
+            gd = morphlib.gitdir.GitDirectory(gitdir)
+            self.assertRaises(IOError,
+                              gd.read_file, 'non-existant-file', 'HEAD')
+
     def test_HEAD(self):
         gd = morphlib.gitdir.GitDirectory(self.dirname)
         self.assertEqual(gd.HEAD, 'master')
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index 92c5e763..9bccb20b 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -225,6 +225,11 @@ class LocalRepoCache(object):
         self.fs.rename(target, path)
         return self.get_repo(reponame)
 
+    def _new_cached_repo_instance(self, reponame, repourl,
+                                  path):  # pragma: no cover
+        return morphlib.cachedrepo.CachedRepo(
+            self._app, reponame, repourl, path)
+
     def get_repo(self, reponame):
         '''Return an object representing a cached repository.'''
 
@@ -234,8 +239,7 @@ class LocalRepoCache(object):
             repourl = self._resolver.pull_url(reponame)
             path = self._cache_name(repourl)
             if self.fs.exists(path):
-                repo = morphlib.cachedrepo.CachedRepo(self._app, reponame,
-                                                      repourl, path)
+                repo = self._new_cached_repo_instance(reponame, repourl, path)
                 self._cached_repo_objects[reponame] = repo
                 return repo
         raise NotCached(reponame)
diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py
index 3cc4f07f..ab6e71fd 100644
--- a/morphlib/localrepocache_tests.py
+++ b/morphlib/localrepocache_tests.py
@@ -22,6 +22,7 @@ import cliapp
 import fs.memoryfs
 
 import morphlib
+import morphlib.gitdir_tests
 
 
 class FakeApplication(object):
@@ -56,6 +57,7 @@ class LocalRepoCacheTests(unittest.TestCase):
         self.lrc._git = self.fake_git
         self.lrc._fetch = self.not_found
         self.lrc._mkdtemp = self.fake_mkdtemp
+        self.lrc._new_cached_repo_instance = self.new_cached_repo_instance
         self._mkdtemp_count = 0
 
     def fake_git(self, args, **kwargs):
@@ -86,6 +88,11 @@ class LocalRepoCacheTests(unittest.TestCase):
         self.lrc.fs.makedir(dirname+"/"+thing)
         return thing
 
+    def new_cached_repo_instance(self, *args):
+        with morphlib.gitdir_tests.allow_nonexistant_git_repos():
+            return morphlib.cachedrepo.CachedRepo(
+                FakeApplication(), *args)
+
     def not_found(self, url, path):
         raise cliapp.AppException('Not found')
 
diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py
index dad7238e..a3ac2749 100644
--- a/morphlib/morphologyfactory.py
+++ b/morphlib/morphologyfactory.py
@@ -57,10 +57,11 @@ class MorphologyFactory(object):
                         chatty=True)
             try:
                 repo = self._lrc.get_repo(reponame)
-                morph = loader.load_from_string(repo.cat(sha1, filename))
+                text = repo.read_file(filename, sha1)
+                morph = loader.load_from_string(text)
             except IOError:
                 morph = None
-                file_list = repo.ls_tree(sha1)
+                file_list = repo.list_files(ref=sha1, recurse=False)
         elif self._rrc is not None:
             self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s"
                         " from the remote git cache.",
diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py
index 41d06480..5222ca6d 100644
--- a/morphlib/morphologyfactory_tests.py
+++ b/morphlib/morphologyfactory_tests.py
@@ -37,6 +37,7 @@ class FakeRemoteRepoCache(object):
     def ls_tree(self, reponame, sha1):
         return []
 
+
 class FakeLocalRepo(object):
 
     morphologies = {
@@ -115,7 +116,7 @@ class FakeLocalRepo(object):
     def __init__(self):
         self.arch = 'x86_64'
 
-    def cat(self, sha1, filename):
+    def read_file(self, filename, ref):
         if filename in self.morphologies:
             values = {
                 'arch': self.arch,
@@ -129,9 +130,10 @@ class FakeLocalRepo(object):
             }''' % filename[:-len('.morph')]
         return 'text'
 
-    def ls_tree(self, sha1):
+    def list_files(self, ref, recurse):
         return self.morphologies.keys()
 
+
 class FakeLocalRepoCache(object):
 
     def __init__(self, lr):
@@ -163,14 +165,14 @@ class MorphologyFactoryTests(unittest.TestCase):
         return ['chunk.morph']
 
     def nolocalmorph(self, *args):
-        if args[-1].endswith('.morph'):
+        if args[0].endswith('.morph'):
             raise IOError('File not found')
         return 'text'
 
-    def autotoolsbuildsystem(self, *args):
+    def autotoolsbuildsystem(self, *args, **kwargs):
         return ['configure.in']
 
-    def remotemorph(self, *args):
+    def remotemorph(self, *args, **kwargs):
         return ['remote-chunk.morph']
 
     def noremotemorph(self, *args):
@@ -182,7 +184,7 @@ class MorphologyFactoryTests(unittest.TestCase):
         return False
 
     def test_gets_morph_from_local_repo(self):
-        self.lr.ls_tree = self.localmorph
+        self.lr.list_files = self.localmorph
         morph = self.mf.get_morphology('reponame', 'sha1',
                                        'chunk.morph')
         self.assertEqual('chunk', morph['name'])
@@ -195,8 +197,8 @@ class MorphologyFactoryTests(unittest.TestCase):
         self.assertEqual('remote-chunk', morph['name'])
 
     def test_autodetects_local_morphology(self):
-        self.lr.cat = self.nolocalmorph
-        self.lr.ls_tree = self.autotoolsbuildsystem
+        self.lr.read_file = self.nolocalmorph
+        self.lr.list_files = self.autotoolsbuildsystem
         morph = self.mf.get_morphology('reponame', 'sha1',
                                        'assumed-local.morph')
         self.assertEqual('assumed-local', morph['name'])
@@ -210,7 +212,7 @@ class MorphologyFactoryTests(unittest.TestCase):
         self.assertEqual('assumed-remote', morph['name'])
 
     def test_raises_error_when_no_local_morph(self):
-        self.lr.cat = self.nolocalfile
+        self.lr.read_file = self.nolocalfile
         self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology,
                           'reponame', 'sha1', 'unreached.morph')
 
@@ -225,14 +227,14 @@ class MorphologyFactoryTests(unittest.TestCase):
                           'reponame', 'sha1', 'name-mismatch.morph')
 
     def test_looks_locally_with_no_remote(self):
-        self.lr.ls_tree = self.localmorph
+        self.lr.list_files = self.localmorph
         morph = self.lmf.get_morphology('reponame', 'sha1',
                                         'chunk.morph')
         self.assertEqual('chunk', morph['name'])
 
     def test_autodetects_locally_with_no_remote(self):
-        self.lr.cat = self.nolocalmorph
-        self.lr.ls_tree = self.autotoolsbuildsystem
+        self.lr.read_file = self.nolocalmorph
+        self.lr.list_files = self.autotoolsbuildsystem
         morph = self.mf.get_morphology('reponame', 'sha1',
                                         'assumed-local.morph')
         self.assertEqual('assumed-local', morph['name'])
diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py
index 74645f41..2d1fe979 100644
--- a/morphlib/plugins/artifact_inspection_plugin.py
+++ b/morphlib/plugins/artifact_inspection_plugin.py
@@ -51,7 +51,7 @@ class ProjectVersionGuesser(object):
             if self.lrc.has_repo(repo):
                 repository = self.lrc.get_repo(repo)
                 for filename in filenames:
-                    yield filename, repository.cat(ref, filename)
+                    yield filename, repository.read_file(filename, ref)
             elif self.rrc:
                 for filename in filenames:
                     yield filename, self.rrc.cat_file(repo, ref, filename)
@@ -153,7 +153,7 @@ class VersionGuesser(object):
                 repository = self.lrc.get_repo(repo)
                 if not self.app.settings['no-git-update']:
                     repository.update()
-                tree = repository.ls_tree(ref)
+                tree = repository.list_files(ref=ref, recurse=False)
             elif self.rrc:
                 repository = None
                 tree = self.rrc.ls_tree(repo, ref)
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index 5531f7f6..76da9736 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -174,7 +174,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
         cached_repo = lrc.get_updated_repo(root_url)
 
         # Check the git branch exists.
-        cached_repo.resolve_ref(system_branch)
+        cached_repo.resolve_ref_to_commit(system_branch)
 
         with self._initializing_system_branch(
             ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
@@ -233,7 +233,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
                 (system_branch, root_url))
 
         # Make sure the base_ref exists.
-        cached_repo.resolve_ref(base_ref)
+        cached_repo.resolve_ref_to_commit(base_ref)
 
         with self._initializing_system_branch(
             ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 8c9fd8b8..3a328eb7 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -71,7 +71,8 @@ class SourceResolver(object):
                 repo.update()
             # If the user passed --no-git-update, and the ref is a SHA1 not
             # available locally, this call will raise an exception.
-            absref, tree = repo.resolve_ref(ref)
+            absref = repo.resolve_ref_to_commit(ref)
+            tree = repo.resolve_ref_to_tree(absref)
         elif self.rrc is not None:
             try:
                 absref, tree = self.rrc.resolve_ref(reponame, ref)
@@ -91,7 +92,8 @@ class SourceResolver(object):
                 repo.update()
             else:
                 repo = self.lrc.get_repo(reponame)
-            absref, tree = repo.resolve_ref(ref)
+            absref = repo.resolve_ref_to_commit(ref)
+            tree = repo.resolve_ref_to_tree(absref)
         return absref, tree
 
     def traverse_morphs(self, definitions_repo, definitions_ref,
@@ -103,7 +105,9 @@ class SourceResolver(object):
         definitions_queue = collections.deque(system_filenames)
         chunk_in_definitions_repo_queue = []
         chunk_in_source_repo_queue = []
-        resolved_refs = {}
+
+        resolved_commits = {}
+        resolved_trees = {}
         resolved_morphologies = {}
 
         # Resolve the (repo, ref) pair for the definitions repo, cache result.
@@ -146,9 +150,12 @@ class SourceResolver(object):
                         (c['repo'], c['ref'], c['morph']))
 
         for repo, ref, filename in chunk_in_definitions_repo_queue:
-            if (repo, ref) not in resolved_refs:
-                resolved_refs[repo, ref] = self.resolve_ref(repo, ref)
-            absref, tree = resolved_refs[repo, ref]
+            if (repo, ref) not in resolved_trees:
+                commit_sha1, tree_sha1 = self.resolve_ref(repo, ref)
+                resolved_commits[repo, ref] = commit_sha1
+                resolved_trees[repo, commit_sha1] = tree_sha1
+            absref = resolved_commits[repo, ref]
+            tree = resolved_trees[repo, absref]
             key = (definitions_repo, definitions_absref, filename)
             if not key in resolved_morphologies:
                 resolved_morphologies[key] = morph_factory.get_morphology(*key)
@@ -156,9 +163,12 @@ class SourceResolver(object):
             visit(repo, ref, filename, absref, tree, morphology)
 
         for repo, ref, filename in chunk_in_source_repo_queue:
-            if (repo, ref) not in resolved_refs:
-                resolved_refs[repo, ref] = self.resolve_ref(repo, ref)
-            absref, tree = resolved_refs[repo, ref]
+            if (repo, ref) not in resolved_trees:
+                commit_sha1, tree_sha1 = self.resolve_ref(repo, ref)
+                resolved_commits[repo, ref] = commit_sha1
+                resolved_trees[repo, commit_sha1] = tree_sha1
+            absref = resolved_commits[repo, ref]
+            tree = resolved_trees[repo, absref]
             key = (repo, absref, filename)
             if key not in resolved_morphologies:
                 resolved_morphologies[key] = morph_factory.get_morphology(*key)
diff --git a/tests.branching/branch-cleans-up-on-failure.stderr b/tests.branching/branch-cleans-up-on-failure.stderr
index 37533408..959226d8 100644
--- a/tests.branching/branch-cleans-up-on-failure.stderr
+++ b/tests.branching/branch-cleans-up-on-failure.stderr
@@ -1 +1 @@
-ERROR: Ref invalid-ref is an invalid reference for repo file://TMP/morphs
+ERROR: Git directory TMP/morphs has no commit at ref invalid-ref^{commit}.
diff --git a/tests.branching/checkout-cleans-up-on-failure.stderr b/tests.branching/checkout-cleans-up-on-failure.stderr
index 5b6a5645..14ed130c 100644
--- a/tests.branching/checkout-cleans-up-on-failure.stderr
+++ b/tests.branching/checkout-cleans-up-on-failure.stderr
@@ -1 +1 @@
-ERROR: Ref i/do/not/exist is an invalid reference for repo file://TMP/morphs
+ERROR: Git directory TMP/morphs has no commit at ref i/do/not/exist^{commit}.
diff --git a/tests.build/missing-ref.stderr b/tests.build/missing-ref.stderr
index 5fa5456b..b5139e25 100644
--- a/tests.build/missing-ref.stderr
+++ b/tests.build/missing-ref.stderr
@@ -1 +1 @@
-ERROR: Ref non-existent-branch is an invalid reference for repo file://TMP/morphs-repo
+ERROR: Git directory TMP/morphs-repo has no commit at ref non-existent-branch^{commit}.