From af069de3dd5e36cf60f1f81a19cca873f92e3dcf Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 28 May 2015 16:29:47 +0100 Subject: Make the python ext use caching This doesn't provide much performance benefit, since most of our queries go through xmlrpclib, but caching here does no harm. Change-Id: Id740a7ffab56defeddb3a6f3f481d81498a4411a --- baserockimport/exts/python.to_lorry | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/baserockimport/exts/python.to_lorry b/baserockimport/exts/python.to_lorry index 2c57a9b..ebde27a 100755 --- a/baserockimport/exts/python.to_lorry +++ b/baserockimport/exts/python.to_lorry @@ -32,20 +32,11 @@ import yaml import pkg_resources +from importer_base import WebServiceClient from importer_python_common import * from utils import warn, error import utils -def fetch_package_metadata(package_name): - try: - result = requests.get('%s/%s/json' % (PYPI_URL, package_name)) - - # raise exception if status code is not 200 OK - result.raise_for_status() - except Exception as e: - error("Couldn't fetch package metadata:", e) - - return result.json() def find_repo_type(url): @@ -162,6 +153,13 @@ class PythonLorryExtension(ImportExtension): def __init__(self): super(PythonLorryExtension, self).__init__() + self.apiclient = WebServiceClient('python_api_cache') + + def fetch_package_metadata(self, package_name): + r = self.apiclient.request('%s/%s/json' % (PYPI_URL, package_name)) + r.raise_for_status() + + return r def run(self): if len(sys.argv) not in [2, 3]: @@ -187,8 +185,12 @@ class PythonLorryExtension(ImportExtension): logging.debug('Treating %s as %s' % (package_name, new_proj_name)) package_name = new_proj_name - metadata = fetch_package_metadata(package_name) - info = metadata['info'] + try: + metadata = self.fetch_package_metadata(package_name) + except Exception as e: + error("Couldn't fetch package metadata: ", e) + + info = metadata.json()['info'] repo_type = (find_repo_type(info['home_page']) if 'home_page' in info else None) -- cgit v1.2.1