Move WebServiceClient into ImporterBase

To allow all extensions to use it,
this also modifies the client so that the cache
gets expired after 5 minutes by default.

The error message returned by the rubygems extension on failure to
get gem data will also be slightly more detailed,

old message,

ERROR: Request to http://rubygems.org/api/v1/gems/kittens.json failed: Not Found

new message,

ERROR: Request to http://rubygems.org/api/v1/gems/kittens.json failed: 404 Client Error: Not Found

Change-Id: I63354fc7682bb01b1122007c1435bf35975db1aa

2 files changed, 37 insertions, 34 deletions

diff --git a/baserockimport/exts/importer_base.py b/baserockimport/exts/importer_base.py
index 5e75f65..e5bc52c 100644
--- a/baserockimport/exts/importer_base.py
+++ b/baserockimport/exts/importer_base.py
@@ -19,6 +19,18 @@
 import logging
 import os
 import sys
+import requests
+import requests_cache
+
+DEFAULT_EXPIRE_AFTER = 300 # seconds
+
+
+class WebServiceClient(object):
+    def __init__(self, cache_name, expire_after=DEFAULT_EXPIRE_AFTER):
+        requests_cache.install_cache(cache_name, expire_after=expire_after)
+
+    def request(self, url):
+        return requests.get(url)
 
 
 class ImportException(Exception):
diff --git a/baserockimport/exts/rubygems.to_lorry b/baserockimport/exts/rubygems.to_lorry
index 0413204..340e889 100755
--- a/baserockimport/exts/rubygems.to_lorry
+++ b/baserockimport/exts/rubygems.to_lorry
@@ -18,53 +18,26 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import requests
-import requests_cache
 import yaml
-
 import logging
 import json
 import os
 import sys
 import urlparse
 
-from importer_base import ImportException, ImportExtension
+from importer_base import ImportException, ImportExtension, WebServiceClient
+
+RUBYGEMS_API_URL = 'http://rubygems.org/api/v1'
 
 
 class GenerateLorryException(ImportException):
     pass
 
 
-class RubyGemsWebServiceClient(object):
-    def __init__(self):
-        # Save hammering the rubygems.org API: 'requests' API calls are
-        # transparently cached in an SQLite database, instead.
-        requests_cache.install_cache('rubygems_api_cache')
-
-    def _request(self, url):
-        r = requests.get(url)
-        if r.ok:
-            return json.loads(r.text)
-        else:
-            raise GenerateLorryException(
-                'Request to %s failed: %s' % (r.url, r.reason))
-
-    def get_gem_info(self, gem_name):
-        info = self._request(
-            'http://rubygems.org/api/v1/gems/%s.json' % gem_name)
-
-        if info['name'] != gem_name:
-            # Sanity check
-            raise GenerateLorryException(
-                 'Received info for Gem "%s", requested "%s"' % info['name'],
-                  gem_name)
-
-        return info
-
-
 class RubyGemLorryGenerator(ImportExtension):
     def __init__(self):
         super(RubyGemLorryGenerator, self).__init__()
+        self.apiclient = WebServiceClient('rubygems_api_cache')
 
         with open(self.local_data_path('rubygems.yaml'), 'r') as f:
             local_data = yaml.load(f.read())
@@ -163,10 +136,28 @@ class RubyGemLorryGenerator(ImportExtension):
             repo_url = repo_url[:-len('.git')]
         return os.path.basename(repo_url)
 
-    def generate_lorry_for_gem(self, gem_name):
-        rubygems_client = RubyGemsWebServiceClient()
+    def get_gem_info(self, gem_name):
 
-        gem_info = rubygems_client.get_gem_info(gem_name)
+        try:
+            r = self.apiclient.request(RUBYGEMS_API_URL + '/gems/%s.json'
+                                       % gem_name)
+
+            r.raise_for_status()
+            info = r.json()
+        except Exception as e:
+            raise GenerateLorryException('Request to %s failed: %s'
+                                         % (r.url, e))
+
+        if info['name'] != gem_name:
+            # Sanity check
+            raise GenerateLorryException(
+                 'Received info for Gem "%s", requested "%s"' % info['name'],
+                  gem_name)
+
+        return info
+
+    def generate_lorry_for_gem(self, gem_name):
+        gem_info = self.get_gem_info(gem_name)
 
         gem_source_url = self.find_upstream_repo_for_gem(gem_name, gem_info)
         logging.info('Got URL <%s> for %s', gem_source_url, gem_name)