Use tarballs instead of bundles

This makes a non-backward-compatible change to morph which switches it to using
tarballs instead of bundles when initialising cached git repositories.  This is
faster because it doesn't require index-pack --fix-thin operations on the
machine running morph.

1 files changed, 40 insertions, 24 deletions

diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index de619770..ae5fa655 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -41,7 +41,7 @@ def quote_url(url):
     ''' Convert URIs to strings that only contain digits, letters, % and _.
 
     NOTE: When changing the code of this function, make sure to also apply
-    the same to the quote_url() function of lorry. Otherwise the git bundles
+    the same to the quote_url() function of lorry. Otherwise the git tarballs
     generated by lorry may no longer be found by morph.
 
     '''
@@ -86,18 +86,18 @@ class LocalRepoCache(object):
     created.
 
     Instead of cloning via a normal 'git clone' directly from the
-    git server, we first try to download a bundle from a url, and
-    if that works, we clone from the bundle.
+    git server, we first try to download a tarball from a url, and
+    if that works, we unpack the tarball.
 
     '''
 
-    def __init__(self, app, cachedir, resolver, bundle_base_url=None):
+    def __init__(self, app, cachedir, resolver, tarball_base_url=None):
         self._app = app
         self._cachedir = cachedir
         self._resolver = resolver
-        if bundle_base_url and not bundle_base_url.endswith('/'):
-            bundle_base_url += '/'  # pragma: no cover
-        self._bundle_base_url = bundle_base_url
+        if tarball_base_url and not tarball_base_url.endswith('/'):
+            tarball_base_url += '/'  # pragma: no cover
+        self._tarball_base_url = tarball_base_url
         self._cached_repo_objects = {}
 
     def _exists(self, filename):  # pragma: no cover
@@ -120,24 +120,34 @@ class LocalRepoCache(object):
 
         self._app.runcmd(['git'] + args, cwd=cwd)
 
+    def _runcmd(self, args, cwd=None):  # pragma: no cover
+        '''Execute a command.
+
+        This is a method of its own so that unit tests can easily override
+        all use of the external git command.
+
+        '''
+
+        self._app.runcmd(args, cwd=cwd)
+
     def _fetch(self, url, filename):  # pragma: no cover
         '''Fetch contents of url into a file.
 
         This method is meant to be overridden by unit tests.
 
         '''
-        self._app.status(msg="Trying to fetch %(bundle)s to seed the cache",
-                         bundle=url,
+        self._app.status(msg="Trying to fetch %(tarball)s to seed the cache",
+                         tarball=url,
                          chatty=True)
         source_handle = None
         try:
             source_handle = urllib2.urlopen(url)
             with open(filename, 'wb') as target_handle:
                 shutil.copyfileobj(source_handle, target_handle)
-            self._app.status(msg="Bundle fetch successful",
+            self._app.status(msg="Tarball fetch successful",
                              chatty=True)
         except Exception, e:
-            self._app.status(msg="Bundle fetch failed: %(reason)s",
+            self._app.status(msg="Tarball fetch failed: %(reason)s",
                              reason=e,
                              chatty=True)
             raise
@@ -177,7 +187,7 @@ class LocalRepoCache(object):
 
         # FIXME: The following is a nicer way than to do this.
         # However, for compatibility, we need to use the same as the
-        # bundle server (set up by Lorry) uses.
+        # tarball server (set up by Lorry) uses.
         # return urllib.quote(url, safe='')
 
         return quote_url(url)
@@ -193,26 +203,32 @@ class LocalRepoCache(object):
         path = self._cache_name(url)
         return self._exists(path)
 
-    def _clone_with_bundle(self, repourl, path):
+    def _clone_with_tarball(self, repourl, path):
         escaped = self._escape(repourl)
-        bundle_url = urlparse.urljoin(self._bundle_base_url, escaped) + '.bndl'
-        bundle_path = path + '.bundle'
+        tarball_url = urlparse.urljoin(self._tarball_base_url,
+                                       escaped) + '.tar'
+        tarball_path = path + '.tar'
 
         try:
-            self._fetch(bundle_url, bundle_path)
+            self._fetch(tarball_url, tarball_path)
         except urllib2.URLError, e:
-            return False, 'Unable to fetch bundle %s: %s' % (bundle_url, e)
+            return False, 'Unable to fetch tarball %s: %s' % (tarball_url, e)
 
         try:
-            self._git(['clone', '--mirror', '-n', bundle_path, path])
-            self._git(['remote', 'set-url', 'origin', repourl], cwd=path)
+            self._mkdir(path)
+            self._runcmd(['tar', 'xf', tarball_path], cwd=path)
+            self._git(['config', 'remote.origin.url', repourl], cwd=path)
+            self._git(['config', 'remote.origin.mirror', 'true'], cwd=path)
+            self._git(['config', 'remote.origin.fetch', '+refs/*:refs/*'],
+                      cwd=path)
         except cliapp.AppException, e:  # pragma: no cover
             if self._exists(path):
                 shutil.rmtree(path)
-            return False, 'Unable to extract bundle %s: %s' % (bundle_path, e)
+            return False, 'Unable to extract tarball %s: %s' % (
+                tarball_path, e)
         finally:
-            if self._exists(bundle_path):
-                self._remove(bundle_path)
+            if self._exists(tarball_path):
+                self._remove(tarball_path)
 
         return True, None
 
@@ -231,10 +247,10 @@ class LocalRepoCache(object):
         except NotCached, e:
             pass
 
-        if self._bundle_base_url:
+        if self._tarball_base_url:
             repourl = self._resolver.pull_url(reponame)
             path = self._cache_name(repourl)
-            ok, error = self._clone_with_bundle(repourl, path)
+            ok, error = self._clone_with_tarball(repourl, path)
             if ok:
                 return self.get_repo(reponame)
             else: