diff options
| -rwxr-xr-x | morph | 44 | ||||
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/remoterepocache.py | 84 | ||||
| -rw-r--r-- | morphlib/remoterepocache_tests.py | 105 |
4 files changed, 225 insertions, 9 deletions
@@ -55,6 +55,10 @@ class Morph(cliapp.Application): 'base URL to download bundles', metavar='URL', default=None) + self.settings.string(['cache-server'], + 'HTTP URL of the morph cache server to use', + metavar='URL', + default=None) self.settings.string(['cachedir'], 'put build results in DIR', metavar='DIR', @@ -141,20 +145,35 @@ class Morph(cliapp.Application): yield args[0], args[1], args[2] args = args[3:] - def _create_source_pool(self, repo_cache, reponame, ref, filename): + def _create_source_pool( + self, local_repo_cache, remote_repo_cache, + reponame, ref, filename): pool = morphlib.sourcepool.SourcePool() queue = collections.deque([(reponame, ref, filename)]) while queue: reponame, ref, filename = queue.popleft() - if self.settings['no-git-update']: - repo = repo_cache.get_repo(reponame) + if local_repo_cache.has_repo(reponame): + repo = local_repo_cache.get_repo(reponame) + if not self.settings['no-git-update']: + repo.update() + absref = repo.resolve_ref(ref) + text = repo.cat(absref, filename) else: - repo = repo_cache.cache_repo(reponame) - repo.update() - absref = repo.resolve_ref(ref) - text = repo.cat(absref, filename) + try: + absref = remote_repo_cache.resolve_ref(reponame, ref) + text = remote_repo_cache.cat_file( + reponame, absref, filename) + except: + if self.settings['no-git-update']: + repo = local_repo_cache.get_repo(reponame) + else: + repo = local_repo_cache.cache_repo(reponame) + repo.update() + absref = repo.resolve_ref(ref) + text = repo.cat(absref, filename) + morphology = morphlib.morph2.Morphology(text) if morphology['kind'] == 'system': for stratum in morphology['strata']: @@ -226,13 +245,20 @@ class Morph(cliapp.Application): cachedir = os.path.join(self.settings['cachedir'], 'gits') baseurls = self.settings['git-base-url'] bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache( + local_repo_cache = morphlib.localrepocache.LocalRepoCache( cachedir, baseurls, bundle_base_url) + if self.settings['cache-server']: + remote_repo_cache = morphlib.remoterepocache.RemoteRepoCache( + self.settings['cache-server'], baseurls) + else: + remote_repo_cache = None # traverse the morphs to list all the sources for repo, ref, filename in self._itertriplets(args): repo, ref, filename = args[:3] - pool = self._create_source_pool(cache, repo, ref, filename) + pool = self._create_source_pool( + local_repo_cache, remote_repo_cache, + repo, ref, filename) env = morphlib.buildenvironment.BuildEnvironment(self.settings) computer = morphlib.cachekeycomputer.CacheKeyComputer(env) diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 859c0699..be3d7c5c 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -40,6 +40,7 @@ import localrepocache import morph2 import morphology import morphologyloader +import remoterepocache import savefile import source import sourcemanager diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py new file mode 100644 index 00000000..fd843682 --- /dev/null +++ b/morphlib/remoterepocache.py @@ -0,0 +1,84 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import json +import urllib2 +import urlparse + + +class ResolveRefError(cliapp.AppException): + + def __init__(self, repo_name, ref): + cliapp.AppException.__init__( + self, 'Failed to resolve ref %s for repo %s' % + (ref, repo_name)) + + +class CatFileError(cliapp.AppException): + + def __init__(self, repo_name, ref, filename): + cliapp.AppException.__init__( + self, 'Failed to cat file %s in ref %s of repo %s' % + (filename, ref, repo_name)) + + +class RemoteRepoCache(object): + + def __init__(self, server_url, base_urls): + self.server_url = server_url + self._base_urls = base_urls + + def _base_iterate(self, repo_name): + for base_url in self._base_urls: + if not base_url.endswith('/'): + base_url += '/' + repo_url = urlparse.urljoin(base_url, repo_name) + yield repo_url + + def resolve_ref(self, repo_name, ref): + for repo_url in self._base_iterate(repo_name): + try: + return self._resolve_ref_for_repo_url(repo_url, ref) + except: + pass + raise ResolveRefError(repo_name, ref) + + def cat_file(self, repo_name, ref, filename): + for repo_url in self._base_iterate(repo_name): + try: + return self._cat_file_for_repo_url(repo_url, ref, filename) + except: + pass + raise CatFileError(repo_name, ref, filename) + + def _resolve_ref_for_repo_url(self, repo_url, ref): # pragma: no cover + data = self._make_request('sha1s?repo=%s&ref=%s' % (repo_url, ref)) + info = json.loads(data) + return info['sha1'] + + def _cat_file_for_repo_url(self, repo_url, ref, + filename): # pragma: no cover + return self._make_request( + 'files?repo=%s&ref=%s&filename=%s' % (repo_url, ref, filename)) + + def _make_request(self, path): + server_url = self.server_url + if not server_url.endswith('/'): + server_url += '/' + url = urlparse.urljoin(server_url, '/1.0/%s' % path) + handle = urllib2.urlopen(url) + return handle.read() diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py new file mode 100644 index 00000000..3e020099 --- /dev/null +++ b/morphlib/remoterepocache_tests.py @@ -0,0 +1,105 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import unittest + +import morphlib + + +class RemoteRepoCacheTests(unittest.TestCase): + + def _resolve_ref_for_repo_url(self, repo_url, ref): + return self.sha1s[repo_url][ref] + + def _cat_file_for_repo_url(self, repo_url, sha1, filename): + return self.files[repo_url][sha1][filename] + + def setUp(self): + self.sha1s = { + 'git://gitorious.org/baserock/morph': { + 'master': 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9' + } + } + self.files = { + 'git://gitorious.org/baserock-morphs/linux': { + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': { + 'linux.morph': 'linux morphology' + } + } + } + self.server_url = 'http://foo.bar' + self.base_urls = [ + 'git://gitorious.org/baserock-morphs', + 'git://gitorious.org/baserock' + ] + self.cache = morphlib.remoterepocache.RemoteRepoCache( + self.server_url, self.base_urls) + self.cache._resolve_ref_for_repo_url = self._resolve_ref_for_repo_url + self.cache._cat_file_for_repo_url = self._cat_file_for_repo_url + + def test_sets_server_url(self): + self.assertEqual(self.cache.server_url, self.server_url) + + def test_resolve_existing_ref_for_existing_repo(self): + sha1 = self.cache.resolve_ref('morph', 'master') + self.assertEqual( + sha1, + self.sha1s['git://gitorious.org/baserock/morph']['master']) + + def test_fail_resolving_existing_ref_for_non_existent_repo(self): + self.assertRaises(morphlib.remoterepocache.ResolveRefError, + self.cache.resolve_ref, 'non-existent-repo', + 'master') + + def test_fail_resolving_non_existent_ref_for_existing_repo(self): + self.assertRaises(morphlib.remoterepocache.ResolveRefError, + self.cache.resolve_ref, 'morph', + 'non-existent-ref') + + def test_fail_resolving_non_existent_ref_for_non_existent_repo(self): + self.assertRaises(morphlib.remoterepocache.ResolveRefError, + self.cache.resolve_ref, 'non-existent-repo', + 'non-existent-ref') + + def test_cat_existing_file_in_existing_repo_and_ref(self): + content = self.cache.cat_file( + 'linux', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'linux.morph') + self.assertEqual(content, 'linux morphology') + + def test_fail_cat_file_using_invalid_sha1(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'linux', 'blablabla', + 'linux.morph') + + def test_fail_cat_non_existent_file_in_existing_repo_and_ref(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'linux', + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'non-existent-file') + + def test_fail_cat_file_in_non_existent_ref_in_existing_repo(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'linux', + 'ecd7a325095a0d19b8c3d76f578d85b979461d41', + 'linux.morph') + + def test_fail_cat_file_in_non_existent_repo(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'non-existent-repo', + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'some-file') + |