From ebed9ace77eed4b8e40675cff40a96f61ae291e2 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Tue, 25 Sep 2012 13:46:02 +0000 Subject: Use tarballs instead of bundles This makes a non-backward-compatible change to morph which switches it to using tarballs instead of bundles when initialising cached git repositories. This is faster because it doesn't require index-pack --fix-thin operations on the machine running morph. --- morphlib/localrepocache.py | 64 +++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 24 deletions(-) (limited to 'morphlib/localrepocache.py') diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index de619770..ae5fa655 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -41,7 +41,7 @@ def quote_url(url): ''' Convert URIs to strings that only contain digits, letters, % and _. NOTE: When changing the code of this function, make sure to also apply - the same to the quote_url() function of lorry. Otherwise the git bundles + the same to the quote_url() function of lorry. Otherwise the git tarballs generated by lorry may no longer be found by morph. ''' @@ -86,18 +86,18 @@ class LocalRepoCache(object): created. Instead of cloning via a normal 'git clone' directly from the - git server, we first try to download a bundle from a url, and - if that works, we clone from the bundle. + git server, we first try to download a tarball from a url, and + if that works, we unpack the tarball. ''' - def __init__(self, app, cachedir, resolver, bundle_base_url=None): + def __init__(self, app, cachedir, resolver, tarball_base_url=None): self._app = app self._cachedir = cachedir self._resolver = resolver - if bundle_base_url and not bundle_base_url.endswith('/'): - bundle_base_url += '/' # pragma: no cover - self._bundle_base_url = bundle_base_url + if tarball_base_url and not tarball_base_url.endswith('/'): + tarball_base_url += '/' # pragma: no cover + self._tarball_base_url = tarball_base_url self._cached_repo_objects = {} def _exists(self, filename): # pragma: no cover @@ -120,24 +120,34 @@ class LocalRepoCache(object): self._app.runcmd(['git'] + args, cwd=cwd) + def _runcmd(self, args, cwd=None): # pragma: no cover + '''Execute a command. + + This is a method of its own so that unit tests can easily override + all use of the external git command. + + ''' + + self._app.runcmd(args, cwd=cwd) + def _fetch(self, url, filename): # pragma: no cover '''Fetch contents of url into a file. This method is meant to be overridden by unit tests. ''' - self._app.status(msg="Trying to fetch %(bundle)s to seed the cache", - bundle=url, + self._app.status(msg="Trying to fetch %(tarball)s to seed the cache", + tarball=url, chatty=True) source_handle = None try: source_handle = urllib2.urlopen(url) with open(filename, 'wb') as target_handle: shutil.copyfileobj(source_handle, target_handle) - self._app.status(msg="Bundle fetch successful", + self._app.status(msg="Tarball fetch successful", chatty=True) except Exception, e: - self._app.status(msg="Bundle fetch failed: %(reason)s", + self._app.status(msg="Tarball fetch failed: %(reason)s", reason=e, chatty=True) raise @@ -177,7 +187,7 @@ class LocalRepoCache(object): # FIXME: The following is a nicer way than to do this. # However, for compatibility, we need to use the same as the - # bundle server (set up by Lorry) uses. + # tarball server (set up by Lorry) uses. # return urllib.quote(url, safe='') return quote_url(url) @@ -193,26 +203,32 @@ class LocalRepoCache(object): path = self._cache_name(url) return self._exists(path) - def _clone_with_bundle(self, repourl, path): + def _clone_with_tarball(self, repourl, path): escaped = self._escape(repourl) - bundle_url = urlparse.urljoin(self._bundle_base_url, escaped) + '.bndl' - bundle_path = path + '.bundle' + tarball_url = urlparse.urljoin(self._tarball_base_url, + escaped) + '.tar' + tarball_path = path + '.tar' try: - self._fetch(bundle_url, bundle_path) + self._fetch(tarball_url, tarball_path) except urllib2.URLError, e: - return False, 'Unable to fetch bundle %s: %s' % (bundle_url, e) + return False, 'Unable to fetch tarball %s: %s' % (tarball_url, e) try: - self._git(['clone', '--mirror', '-n', bundle_path, path]) - self._git(['remote', 'set-url', 'origin', repourl], cwd=path) + self._mkdir(path) + self._runcmd(['tar', 'xf', tarball_path], cwd=path) + self._git(['config', 'remote.origin.url', repourl], cwd=path) + self._git(['config', 'remote.origin.mirror', 'true'], cwd=path) + self._git(['config', 'remote.origin.fetch', '+refs/*:refs/*'], + cwd=path) except cliapp.AppException, e: # pragma: no cover if self._exists(path): shutil.rmtree(path) - return False, 'Unable to extract bundle %s: %s' % (bundle_path, e) + return False, 'Unable to extract tarball %s: %s' % ( + tarball_path, e) finally: - if self._exists(bundle_path): - self._remove(bundle_path) + if self._exists(tarball_path): + self._remove(tarball_path) return True, None @@ -231,10 +247,10 @@ class LocalRepoCache(object): except NotCached, e: pass - if self._bundle_base_url: + if self._tarball_base_url: repourl = self._resolver.pull_url(reponame) path = self._cache_name(repourl) - ok, error = self._clone_with_bundle(repourl, path) + ok, error = self._clone_with_tarball(repourl, path) if ok: return self.get_repo(reponame) else: -- cgit v1.2.1