diff options
| -rw-r--r-- | morphlib/localrepocache.py | 49 | ||||
| -rw-r--r-- | morphlib/localrepocache_tests.py | 26 |
2 files changed, 72 insertions, 3 deletions
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 673f9854..9beba4af 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -66,11 +66,17 @@ class LocalRepoCache(object): is prepended. The base urls are given to the class when it is created. + Instead of cloning via a normal 'git clone' directly from the + git server, we first try to download a bundle from a url, and + if that works, we clone from the bundle. + ''' - def __init__(self, cachedir, baseurls): + def __init__(self, cachedir, baseurls, bundle_base_url): + assert bundle_base_url.endswith('/') self._cachedir = cachedir self._baseurls = baseurls + self._bundle_base_url = bundle_base_url self._ex = morphlib.execute.Execute(cachedir, logging.debug) def _exists(self, filename): # pragma: no cover @@ -93,9 +99,34 @@ class LocalRepoCache(object): self._ex.runv(['git'] + args) + def _fetch(self, url, filename): # pragma: no cover + '''Fetch contents of url into a file. + + This method is meant to be overridden by unit tests. + + ''' + + source_handle = urllib2.urlopen(url) + target_handle = open(filename, 'wb') + + data = source_handle.read(4096) + while data: + target_handle.write(data) + data = source_handle.read(4096) + + source_handle.close() + target_handle.close() + def _escape(self, url): '''Escape a URL so it can be used as a basename in a file.''' - return urllib.quote(url, safe='') + + # FIXME: The following is a nicer way than what source manager does. + # However, for compatibility, we need to use the same as the source + # manager uses, since that's what the bundle server (set up by + # Lorry) uses. + # return urllib.quote(url, safe='') + + return morphlib.sourcemanager.quote_url(url) def _cache_name(self, url): basename = self._escape(url) @@ -115,6 +146,16 @@ class LocalRepoCache(object): return True return False + def _clone_with_bundle(self, repourl, path): + escaped = self._escape(repourl) + bundle_url = urlparse.urljoin(self._bundle_base_url, escaped) + bundle_path = path + '.bundle' + if self._fetch(bundle_url, bundle_path): + self._git(['clone', bundle_path, path]) + return True + else: + return False + def cache_repo(self, reponame): '''Clone the given repo into the cache. @@ -125,6 +166,10 @@ class LocalRepoCache(object): for repourl, path in self._base_iterate(reponame): if self._exists(path): break + + if self._clone_with_bundle(repourl, path): + break + try: self._git(['clone', reponame, path]) except morphlib.execute.CommandFailure: diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index 13a1c415..add0aa20 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -23,15 +23,22 @@ class LocalRepoCacheTests(unittest.TestCase): def setUp(self): baseurls = ['git://example.com/'] + bundle_base_url = 'http://lorry.example.com/bundles/' self.reponame = 'reponame' self.repourl = 'git://example.com/reponame' + escaped_url = 'git___example_com_reponame' + self.bundle_url = '%s%s' % (bundle_base_url, escaped_url) self.cachedir = '/cache/dir' + self.cache_path = '%s/%s' % (self.cachedir, escaped_url) self.cache = set() self.remotes = [] + self.fetched = [] self.lrc = morphlib.localrepocache.LocalRepoCache(self.cachedir, - baseurls) + baseurls, + bundle_base_url) self.lrc._git = self.fake_git self.lrc._exists = self.fake_exists + self.lrc._fetch = self.not_found def fake_git(self, args): if args[0] == 'clone': @@ -48,6 +55,13 @@ class LocalRepoCacheTests(unittest.TestCase): def fake_exists(self, filename): return filename in self.cache + def not_found(self, url, path): + return False + + def fake_fetch(self, url, path): + self.fetched.append(url) + return True + def test_has_not_got_relative_repo_initially(self): self.assertFalse(self.lrc.has_repo(self.reponame)) @@ -75,6 +89,16 @@ class LocalRepoCacheTests(unittest.TestCase): self.assertRaises(morphlib.localrepocache.NoRemote, self.lrc.cache_repo, self.repourl) + def test_does_not_mind_a_missing_bundle(self): + self.lrc.cache_repo(self.repourl) + self.assertEqual(self.fetched, []) + + def test_fetches_bundle_when_it_exists(self): + self.lrc._fetch = self.fake_fetch + self.lrc.cache_repo(self.repourl) + self.assertEqual(self.fetched, [self.bundle_url]) + self.assertEqual(self.remotes, [self.cache_path + '.bundle']) + def test_gets_cached_relative_repo(self): self.lrc.cache_repo(self.reponame) cached = self.lrc.get_repo(self.reponame) |