From 5eb0ee253316dc930a4f53a8366764c280ae8fe5 Mon Sep 17 00:00:00 2001 From: Jannis Pohlmann Date: Wed, 18 Apr 2012 13:29:17 +0100 Subject: Add RemoteRepoCache and integrate it into _create_source_pool(). This adds a new setting called 'cache-server' to morph. It is None by default and should be set to the HTTP URL of a morph cache server to be used. The RemoteRepoCache object provides two methods: resolve_ref() and cat_file(), both of which wrap the communication with the cache server and return a SHA1 string and file contents, respectively. The _create_source_pool() method now takes a local and an optional remote repo cache and tries to do whatever is best to resolve refs and load morphologies for the Source objects it creates. --- morphlib/__init__.py | 1 + morphlib/remoterepocache.py | 84 ++++++++++++++++++++++++++++++ morphlib/remoterepocache_tests.py | 105 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 morphlib/remoterepocache.py create mode 100644 morphlib/remoterepocache_tests.py (limited to 'morphlib') diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 859c0699..be3d7c5c 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -40,6 +40,7 @@ import localrepocache import morph2 import morphology import morphologyloader +import remoterepocache import savefile import source import sourcemanager diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py new file mode 100644 index 00000000..fd843682 --- /dev/null +++ b/morphlib/remoterepocache.py @@ -0,0 +1,84 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import json +import urllib2 +import urlparse + + +class ResolveRefError(cliapp.AppException): + + def __init__(self, repo_name, ref): + cliapp.AppException.__init__( + self, 'Failed to resolve ref %s for repo %s' % + (ref, repo_name)) + + +class CatFileError(cliapp.AppException): + + def __init__(self, repo_name, ref, filename): + cliapp.AppException.__init__( + self, 'Failed to cat file %s in ref %s of repo %s' % + (filename, ref, repo_name)) + + +class RemoteRepoCache(object): + + def __init__(self, server_url, base_urls): + self.server_url = server_url + self._base_urls = base_urls + + def _base_iterate(self, repo_name): + for base_url in self._base_urls: + if not base_url.endswith('/'): + base_url += '/' + repo_url = urlparse.urljoin(base_url, repo_name) + yield repo_url + + def resolve_ref(self, repo_name, ref): + for repo_url in self._base_iterate(repo_name): + try: + return self._resolve_ref_for_repo_url(repo_url, ref) + except: + pass + raise ResolveRefError(repo_name, ref) + + def cat_file(self, repo_name, ref, filename): + for repo_url in self._base_iterate(repo_name): + try: + return self._cat_file_for_repo_url(repo_url, ref, filename) + except: + pass + raise CatFileError(repo_name, ref, filename) + + def _resolve_ref_for_repo_url(self, repo_url, ref): # pragma: no cover + data = self._make_request('sha1s?repo=%s&ref=%s' % (repo_url, ref)) + info = json.loads(data) + return info['sha1'] + + def _cat_file_for_repo_url(self, repo_url, ref, + filename): # pragma: no cover + return self._make_request( + 'files?repo=%s&ref=%s&filename=%s' % (repo_url, ref, filename)) + + def _make_request(self, path): + server_url = self.server_url + if not server_url.endswith('/'): + server_url += '/' + url = urlparse.urljoin(server_url, '/1.0/%s' % path) + handle = urllib2.urlopen(url) + return handle.read() diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py new file mode 100644 index 00000000..3e020099 --- /dev/null +++ b/morphlib/remoterepocache_tests.py @@ -0,0 +1,105 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import unittest + +import morphlib + + +class RemoteRepoCacheTests(unittest.TestCase): + + def _resolve_ref_for_repo_url(self, repo_url, ref): + return self.sha1s[repo_url][ref] + + def _cat_file_for_repo_url(self, repo_url, sha1, filename): + return self.files[repo_url][sha1][filename] + + def setUp(self): + self.sha1s = { + 'git://gitorious.org/baserock/morph': { + 'master': 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9' + } + } + self.files = { + 'git://gitorious.org/baserock-morphs/linux': { + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': { + 'linux.morph': 'linux morphology' + } + } + } + self.server_url = 'http://foo.bar' + self.base_urls = [ + 'git://gitorious.org/baserock-morphs', + 'git://gitorious.org/baserock' + ] + self.cache = morphlib.remoterepocache.RemoteRepoCache( + self.server_url, self.base_urls) + self.cache._resolve_ref_for_repo_url = self._resolve_ref_for_repo_url + self.cache._cat_file_for_repo_url = self._cat_file_for_repo_url + + def test_sets_server_url(self): + self.assertEqual(self.cache.server_url, self.server_url) + + def test_resolve_existing_ref_for_existing_repo(self): + sha1 = self.cache.resolve_ref('morph', 'master') + self.assertEqual( + sha1, + self.sha1s['git://gitorious.org/baserock/morph']['master']) + + def test_fail_resolving_existing_ref_for_non_existent_repo(self): + self.assertRaises(morphlib.remoterepocache.ResolveRefError, + self.cache.resolve_ref, 'non-existent-repo', + 'master') + + def test_fail_resolving_non_existent_ref_for_existing_repo(self): + self.assertRaises(morphlib.remoterepocache.ResolveRefError, + self.cache.resolve_ref, 'morph', + 'non-existent-ref') + + def test_fail_resolving_non_existent_ref_for_non_existent_repo(self): + self.assertRaises(morphlib.remoterepocache.ResolveRefError, + self.cache.resolve_ref, 'non-existent-repo', + 'non-existent-ref') + + def test_cat_existing_file_in_existing_repo_and_ref(self): + content = self.cache.cat_file( + 'linux', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'linux.morph') + self.assertEqual(content, 'linux morphology') + + def test_fail_cat_file_using_invalid_sha1(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'linux', 'blablabla', + 'linux.morph') + + def test_fail_cat_non_existent_file_in_existing_repo_and_ref(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'linux', + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'non-existent-file') + + def test_fail_cat_file_in_non_existent_ref_in_existing_repo(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'linux', + 'ecd7a325095a0d19b8c3d76f578d85b979461d41', + 'linux.morph') + + def test_fail_cat_file_in_non_existent_repo(self): + self.assertRaises(morphlib.remoterepocache.CatFileError, + self.cache.cat_file, 'non-existent-repo', + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'some-file') + -- cgit v1.2.1