Add RemoteRepoCache and integrate it into _create_source_pool().

This adds a new setting called 'cache-server' to morph. It is None
by default and should be set to the HTTP URL of a morph cache server
to be used.

The RemoteRepoCache object provides two methods: resolve_ref() and
cat_file(), both of which wrap the communication with the cache
server and return a SHA1 string and file contents, respectively.

The _create_source_pool() method now takes a local and an optional
remote repo cache and tries to do whatever is best to resolve
refs and load morphologies for the Source objects it creates.

3 files changed, 190 insertions, 0 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 859c0699..be3d7c5c 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -40,6 +40,7 @@ import localrepocache
 import morph2
 import morphology
 import morphologyloader
+import remoterepocache
 import savefile
 import source
 import sourcemanager
diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py
new file mode 100644
index 00000000..fd843682
--- /dev/null
+++ b/morphlib/remoterepocache.py
@@ -0,0 +1,84 @@
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import json
+import urllib2
+import urlparse
+
+
+class ResolveRefError(cliapp.AppException):
+
+    def __init__(self, repo_name, ref):
+        cliapp.AppException.__init__(
+                self, 'Failed to resolve ref %s for repo %s' %
+                (ref, repo_name))
+
+
+class CatFileError(cliapp.AppException):
+
+    def __init__(self, repo_name, ref, filename):
+        cliapp.AppException.__init__(
+                self, 'Failed to cat file %s in ref %s of repo %s' %
+                (filename, ref, repo_name))
+
+
+class RemoteRepoCache(object):
+
+    def __init__(self, server_url, base_urls):
+        self.server_url = server_url
+        self._base_urls = base_urls
+
+    def _base_iterate(self, repo_name):
+        for base_url in self._base_urls:
+            if not base_url.endswith('/'):
+                base_url += '/'
+            repo_url = urlparse.urljoin(base_url, repo_name)
+            yield repo_url
+
+    def resolve_ref(self, repo_name, ref):
+        for repo_url in self._base_iterate(repo_name):
+            try:
+                return self._resolve_ref_for_repo_url(repo_url, ref)
+            except:
+                pass
+        raise ResolveRefError(repo_name, ref)
+
+    def cat_file(self, repo_name, ref, filename):
+        for repo_url in self._base_iterate(repo_name):
+            try:
+                return self._cat_file_for_repo_url(repo_url, ref, filename)
+            except:
+                pass
+        raise CatFileError(repo_name, ref, filename)
+
+    def _resolve_ref_for_repo_url(self, repo_url, ref): # pragma: no cover
+        data = self._make_request('sha1s?repo=%s&ref=%s' % (repo_url, ref))
+        info = json.loads(data)
+        return info['sha1']
+
+    def _cat_file_for_repo_url(self, repo_url, ref,
+                               filename): # pragma: no cover
+        return self._make_request(
+                'files?repo=%s&ref=%s&filename=%s' % (repo_url, ref, filename))
+
+    def _make_request(self, path):
+        server_url = self.server_url
+        if not server_url.endswith('/'):
+            server_url += '/'
+        url = urlparse.urljoin(server_url, '/1.0/%s' % path)
+        handle = urllib2.urlopen(url)
+        return handle.read()
diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py
new file mode 100644
index 00000000..3e020099
--- /dev/null
+++ b/morphlib/remoterepocache_tests.py
@@ -0,0 +1,105 @@
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import unittest
+
+import morphlib
+
+
+class RemoteRepoCacheTests(unittest.TestCase):
+
+    def _resolve_ref_for_repo_url(self, repo_url, ref):
+        return self.sha1s[repo_url][ref]
+
+    def _cat_file_for_repo_url(self, repo_url, sha1, filename):
+        return self.files[repo_url][sha1][filename]
+    
+    def setUp(self):
+        self.sha1s = {
+            'git://gitorious.org/baserock/morph': {
+                'master': 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9'
+            }
+        }
+        self.files = {
+            'git://gitorious.org/baserock-morphs/linux': {
+                'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': {
+                    'linux.morph': 'linux morphology'
+                }
+            }
+        }
+        self.server_url = 'http://foo.bar'
+        self.base_urls = [
+            'git://gitorious.org/baserock-morphs',
+            'git://gitorious.org/baserock'
+        ]
+        self.cache = morphlib.remoterepocache.RemoteRepoCache(
+                self.server_url, self.base_urls)
+        self.cache._resolve_ref_for_repo_url = self._resolve_ref_for_repo_url
+        self.cache._cat_file_for_repo_url = self._cat_file_for_repo_url
+
+    def test_sets_server_url(self):
+        self.assertEqual(self.cache.server_url, self.server_url)
+
+    def test_resolve_existing_ref_for_existing_repo(self):
+        sha1 = self.cache.resolve_ref('morph', 'master')
+        self.assertEqual(
+                sha1,
+                self.sha1s['git://gitorious.org/baserock/morph']['master'])
+
+    def test_fail_resolving_existing_ref_for_non_existent_repo(self):
+        self.assertRaises(morphlib.remoterepocache.ResolveRefError,
+                          self.cache.resolve_ref, 'non-existent-repo',
+                          'master')
+
+    def test_fail_resolving_non_existent_ref_for_existing_repo(self):
+        self.assertRaises(morphlib.remoterepocache.ResolveRefError,
+                          self.cache.resolve_ref, 'morph',
+                          'non-existent-ref')
+
+    def test_fail_resolving_non_existent_ref_for_non_existent_repo(self):
+        self.assertRaises(morphlib.remoterepocache.ResolveRefError,
+                          self.cache.resolve_ref, 'non-existent-repo',
+                          'non-existent-ref')
+
+    def test_cat_existing_file_in_existing_repo_and_ref(self):
+        content = self.cache.cat_file(
+                'linux', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
+                'linux.morph')
+        self.assertEqual(content, 'linux morphology')
+    
+    def test_fail_cat_file_using_invalid_sha1(self):
+        self.assertRaises(morphlib.remoterepocache.CatFileError,
+                          self.cache.cat_file, 'linux', 'blablabla',
+                          'linux.morph')
+
+    def test_fail_cat_non_existent_file_in_existing_repo_and_ref(self):
+        self.assertRaises(morphlib.remoterepocache.CatFileError,
+                          self.cache.cat_file, 'linux',
+                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
+                          'non-existent-file')
+
+    def test_fail_cat_file_in_non_existent_ref_in_existing_repo(self):
+        self.assertRaises(morphlib.remoterepocache.CatFileError,
+                          self.cache.cat_file, 'linux',
+                          'ecd7a325095a0d19b8c3d76f578d85b979461d41',
+                          'linux.morph')
+
+    def test_fail_cat_file_in_non_existent_repo(self):
+        self.assertRaises(morphlib.remoterepocache.CatFileError,
+                          self.cache.cat_file, 'non-existent-repo',
+                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
+                          'some-file')
+