Rework git caches to be bare mirrors of the repos.

This reworks the code for managing and using the git caches in morph to treat
the caches as bare repositories which are mirrors of where we clone from.  In
addition we correctly prune the branches during updates, so that we don't end
up accumulating pointless branches over and over.  This is even more important
with branch-and-merge generating temporary build refs for things.

5 files changed, 107 insertions, 23 deletions

diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 0b986540..72165ce7 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -308,16 +308,14 @@ class ChunkBuilder(BuilderBase):
 
         cache_dir = os.path.dirname(self.artifact.source.repo.path)
 
-        def extract_repo(path, sha1, destdir):
+        def extract_repo(repo, sha1, destdir):
             self.app.status(msg='Extracting %(source)s into %(target)s',
-                            source=path,
+                            source=repo.original_name,
                             target=destdir)
-            if not os.path.exists(destdir):
-                os.mkdir(destdir)
-            morphlib.git.copy_repository(self.app.runcmd, path, destdir)
-            morphlib.git.checkout_ref(self.app.runcmd, destdir, sha1)
+
+            repo.checkout(sha1, destdir)
             morphlib.git.reset_workdir(self.app.runcmd, destdir)
-            submodules = morphlib.git.Submodules(self.app, path, sha1)
+            submodules = morphlib.git.Submodules(self.app, repo.path, sha1)
             try:
                 submodules.load()
             except morphlib.git.NoModulesFileError:
@@ -327,14 +325,14 @@ class ChunkBuilder(BuilderBase):
                 for sub in submodules:
                     cached_repo = self.repo_cache.get_repo(sub.url)
                     sub_dir = os.path.join(destdir, sub.path)
-                    tuples.append((cached_repo.path, sub.commit, sub_dir))
+                    tuples.append((cached_repo, sub.commit, sub_dir))
                 return tuples
 
         s = self.artifact.source
-        todo = [(s.repo.path, s.sha1, srcdir)]
+        todo = [(s.repo, s.sha1, srcdir)]
         while todo:
-            path, sha1, srcdir = todo.pop()
-            todo += extract_repo(path, sha1, srcdir)
+            repo, sha1, srcdir = todo.pop()
+            todo += extract_repo(repo, sha1, srcdir)
         self.set_mtime_recursively(srcdir)
 
     def set_mtime_recursively(self, root):  # pragma: no cover
diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py
index ad17785a..61526cd7 100644
--- a/morphlib/cachedrepo.py
+++ b/morphlib/cachedrepo.py
@@ -49,6 +49,14 @@ class CloneError(cliapp.AppException):
     def __init__(self, repo, target_dir):
         cliapp.AppException.__init__(
             self,
+            'Failed to clone %s into %s' % (repo.original_name, target_dir))
+
+
+class CopyError(cliapp.AppException):
+
+    def __init__(self, repo, target_dir):
+        cliapp.AppException.__init__(
+            self,
             'Failed to copy %s into %s' % (repo.original_name, target_dir))
 
 
@@ -75,6 +83,9 @@ class CachedRepo(object):
     remote Git repository. This remote repository is set up as the
     'origin' remote.
 
+    Cached repositories are bare mirrors of the upstream.  Locally created
+    branches will be lost the next time the repository updates.
+
     CachedRepo objects can resolve Git refs into SHA1s. Given a SHA1
     ref, they can also be asked to return the contents of a file via the
     cat() method. They can furthermore check out the repository into
@@ -109,8 +120,7 @@ class CachedRepo(object):
         if not self.is_valid_sha1(ref):
             try:
                 refs = self._show_ref(ref).split('\n')
-                # split each ref line into an array, drop non-origin branches
-                refs = [x.split() for x in refs if 'origin' in x]
+                refs = [x.split() for x in refs]
                 absref = refs[0][0]
             except cliapp.AppException:
                 raise InvalidReferenceError(self, ref)
@@ -146,8 +156,8 @@ class CachedRepo(object):
             raise IOError('File %s does not exist in ref %s of repo %s' %
                           (filename, ref, self))
 
-    def checkout(self, ref, target_dir):
-        '''Unpacks the repository in a directory and checks out a commit ref.
+    def clone_checkout(self, ref, target_dir):
+        '''Clone from the cache into the target path and check out a given ref.
 
         Raises a CheckoutDirectoryExistsError if the target
         directory already exists. Raises an InvalidReferenceError if the
@@ -160,9 +170,28 @@ class CachedRepo(object):
         if os.path.exists(target_dir):
             raise CheckoutDirectoryExistsError(self, target_dir)
 
-        os.mkdir(target_dir)
+        self.resolve_ref(ref)
+
+        self._clone_into(target_dir, ref)
+
+    def checkout(self, ref, target_dir):
+        '''Unpacks the repository in a directory and checks out a commit ref.
+
+        Raises an InvalidReferenceError if the ref is not found in the
+        repository. Raises a CopyError if something goes wrong with the copy
+        of the repository. Raises a CheckoutError if something else goes wrong
+        while copying the repository or checking out the SHA1 ref.
+
+        '''
+
+        if not os.path.exists(target_dir):
+            os.mkdir(target_dir)
 
+        # Note, we copy instead of cloning because it's much faster in the case
+        # that the target is on a different filesystem from the cache. We then
+        # take care to turn the copy into something as good as a real clone.
         self._copy_repository(self.path, target_dir)
+
         self._checkout_ref(ref, target_dir)
 
     def ls_tree(self, ref):
@@ -219,11 +248,18 @@ class CachedRepo(object):
         return self._runcmd(['git', 'cat-file', 'blob',
                              '%s:%s' % (ref, filename)])
 
+    def _clone_into(self, target_dir, ref):  #pragma: no cover
+        '''Actually perform the clone'''
+        try:
+            morphlib.git.clone_into(self._runcmd, self.path, target_dir, ref)
+        except cliapp.AppException:
+            raise CloneError(self, target_dir)
+
     def _copy_repository(self, source_dir, target_dir):  # pragma: no cover
         try:
             morphlib.git.copy_repository(self._runcmd, source_dir, target_dir)
         except cliapp.AppException:
-            raise CloneError(self, target_dir)
+            raise CopyError(self, target_dir)
 
     def _checkout_ref(self, ref, target_dir):  # pragma: no cover
         try:
@@ -232,7 +268,7 @@ class CachedRepo(object):
             raise CheckoutError(self, ref, target_dir)
 
     def _update(self):  # pragma: no cover
-        self._runcmd(['git', 'remote', 'update', 'origin'])
+        self._runcmd(['git', 'remote', 'update', 'origin', '--prune'])
 
     def __str__(self):  # pragma: no cover
         return self.url
diff --git a/morphlib/git.py b/morphlib/git.py
index d8169c93..9ab98ad0 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -154,8 +154,42 @@ def set_remote(runcmd, gitdir, name, url):
 
 
 def copy_repository(runcmd, repo, destdir):
-    '''Copies a cached repository into a directory using cp.'''
-    return runcmd(['cp', '-a', os.path.join(repo, '.git'), destdir])
+    '''Copies a cached repository into a directory using cp.
+
+    This also fixes up the repository afterwards, so that it can contain
+    code etc.  It does not leave any given branch ready for use.
+
+    '''
+    runcmd(['cp', '-a', repo, os.path.join(destdir, '.git')])
+    # core.bare should be false so that git believes work trees are possible
+    runcmd(['git', 'config', 'core.bare', 'false'], cwd=destdir)
+    # we do not want the origin remote to behave as a mirror for pulls
+    runcmd(['git', 'config', '--unset', 'remote.origin.mirror'], cwd=destdir)
+    # we want a traditional refs/heads -> refs/remotes/origin ref mapping
+    runcmd(['git', 'config', 'remote.origin.fetch',
+            '+refs/heads/*:refs/remotes/origin/*'], cwd=destdir)
+    # set the origin url to the cached repo so that we can quickly clean up
+    runcmd(['git', 'config', 'remote.origin.url', repo], cwd=destdir)
+    # by packing the refs, we can then edit then en-masse easily
+    runcmd(['git', 'pack-refs', '--all', '--prune'], cwd=destdir)
+    # turn refs/heads/* into refs/remotes/origin/* in the packed refs
+    # so that the new copy behaves more like a traditional clone.
+    logging.debug("Adjusting packed refs for %s" % destdir)
+    with open(os.path.join(destdir, ".git", "packed-refs"), "r") as ref_fh:
+        pack_lines = ref_fh.read().split("\n")
+    with open(os.path.join(destdir, ".git", "packed-refs"), "w") as ref_fh:
+        ref_fh.write(pack_lines.pop(0) + "\n")
+        for refline in pack_lines:
+            if ' refs/remotes/' in refline:
+                continue
+            if ' refs/heads/' in refline:
+                sha, ref = refline[:40], refline[41:]
+                if ref.startswith("refs/heads/"):
+                    ref = "refs/remotes/origin/" + ref[11:]
+                refline = "%s %s" % (sha, ref)
+            ref_fh.write("%s\n" % (refline))
+    # Finally run a remote update to clear up the refs ready for use.
+    runcmd(['git', 'remote', 'update', 'origin', '--prune'], cwd=destdir)
 
 
 def checkout_ref(runcmd, gitdir, ref):
@@ -167,3 +201,16 @@ def reset_workdir(runcmd, gitdir):
     '''and the status of the working directory'''
     runcmd(['git', 'clean', '-fxd'], cwd=gitdir)
     runcmd(['git', 'reset', '--hard', 'HEAD'], cwd=gitdir)
+
+
+def clone_into(runcmd, srcpath, targetpath, ref=None):
+    '''Clones a repo in srcpath into targetpath, optionally directly at ref.'''
+    if ref is None:
+        runcmd(['git', 'clone', srcpath, targetpath])
+    else:
+        runcmd(['git', 'clone', '-b', ref, srcpath, targetpath])
+
+def find_first_ref(runcmd, gitdir, ref):
+    '''Find the *first* ref match and returns its sha1.'''
+    return runcmd(['git', 'show-ref', ref],
+                  cwd=gitdir).split("\n")[0].split(" ")[0]
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index c5a95ebc..89be56c8 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -197,7 +197,7 @@ class LocalRepoCache(object):
             return False, 'Unable to fetch bundle %s: %s' % (bundle_url, e)
 
         try:
-            self._git(['clone', '-n', bundle_path, path])
+            self._git(['clone', '--mirror', '-n', bundle_path, path])
             self._git(['remote', 'set-url', 'origin', repourl], cwd=path)
         except cliapp.AppException, e:  # pragma: no cover
             if self._exists(path):
@@ -236,7 +236,7 @@ class LocalRepoCache(object):
         repourl = self._resolver.pull_url(reponame)
         path = self._cache_name(repourl)
         try:
-            self._git(['clone', '-n', repourl, path])
+            self._git(['clone', '--mirror', '-n', repourl, path])
         except cliapp.AppException, e:
             errors.append('Unable to clone from %s to %s: %s' %
                           (repourl, path, e))
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index d3aced4b..82d83a58 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -194,7 +194,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
             os.makedirs(parent_dir)
 
         # Clone it from cache to target directory.
-        repo.checkout(ref, os.path.abspath(dirname))
+        repo.clone_checkout(ref, os.path.abspath(dirname))
 
         # Remember the repo name we cloned from in order to be able
         # to identify the repo again later using the same name, even
@@ -221,6 +221,9 @@ class BranchAndMergePlugin(cliapp.Plugin):
             with open(filename) as f:
                 text = f.read()
         else:
+            ref = morphlib.git.find_first_ref(self.app.runcmd, repo_dir, ref)
+            logging.warning("Running git cat-file blob %s:%s.morph in %s" % (
+                    ref, name, repo_dir))
             text = self.app.runcmd(['git', 'cat-file', 'blob',
                                    '%s:%s.morph' % (ref, name)], cwd=repo_dir)
         morphology = morphlib.morph2.Morphology(text)