diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2015-02-10 10:34:57 +0000 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2015-02-11 16:04:44 +0000 |
commit | 7a41668f803641af0edd5948125a66d45906d6c7 (patch) | |
tree | 8a564eecd9341bc71407a1de674ab756c1b14683 | |
parent | 3a67c90baf7abecd62b92cf64176a04f337da5bc (diff) | |
download | morph-7a41668f803641af0edd5948125a66d45906d6c7.tar.gz |
Use Requests instead of urllib2 for remote HTTP requests
Requests is a much better library for this purpose.
-rw-r--r-- | morphlib/localrepocache_tests.py | 3 | ||||
-rw-r--r-- | morphlib/remoteartifactcache.py | 31 | ||||
-rw-r--r-- | morphlib/remoteartifactcache_tests.py | 8 | ||||
-rw-r--r-- | morphlib/remoterepocache.py | 18 | ||||
-rw-r--r-- | morphlib/remoterepocache_tests.py | 10 |
5 files changed, 39 insertions, 31 deletions
diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index ab6e71fd..961e06b4 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +15,6 @@ import unittest -import urllib2 import os import cliapp diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index c5bf9a48..73f9cbed 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -16,16 +16,10 @@ import cliapp import logging -import shutil import urllib -import urllib2 import urlparse - -class HeadRequest(urllib2.Request): - - def get_method(self): - return 'HEAD' +import requests class GetError(cliapp.AppException): @@ -54,12 +48,15 @@ class RemoteArtifactCache(object): def _has_file(self, filename): url = self._request_url(filename) logging.debug('RemoteArtifactCache._has_file: url=%s' % url) - request = HeadRequest(url) - try: - urllib2.urlopen(request) - return True - except (urllib2.HTTPError, urllib2.URLError): + + response = requests.head(url) + + if response.status_code == 404: return False + elif response.ok: + return True + else: + response.raise_for_status() def _request_url(self, filename): server_url = self.server_url @@ -73,13 +70,17 @@ class RemoteArtifactCache(object): return self.server_url def _fetch_file(self, remote_filename, local_file): + chunk_size = 1024 * 100 + remote_url = self._request_url(remote_filename) logging.debug('RemoteArtifactCache._fetch_file: url=%s' % remote_url) try: - remote_file = urllib2.urlopen(remote_url) - shutil.copyfileobj(remote_file, local_file) - except (urllib2.HTTPError, urllib2.URLError) as e: + response = requests.get(remote_url, stream=True) + response.raise_for_status() + for chunk in response.iter_content(chunk_size): + local_file.write(chunk) + except requests.exceptions.HTTPError as e: logging.debug(str(e)) raise GetError(self, remote_filename, e) diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index f6d19ae7..fc9b6b27 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -68,7 +68,7 @@ def artifact_cache_server(valid_filenames): pass def artifacts(self, query, send_body=True): - '''Return a cached artifact, or 404.''' + '''Return a cached artifact, or a 404 (not found) error.''' filename = urlparse.parse_qs(query)['filename'][0] if filename in valid_filenames: @@ -142,9 +142,9 @@ class RemoteArtifactCacheTests(unittest.TestCase): rac.has_source_metadata(chunk, chunk.cache_key, 'meta')) lac = morphlib.testutils.FakeLocalArtifactCache() - self.assertRaises( - morphlib.remoteartifactcache.GetError, rac.get_artifacts, - chunk.artifacts.values(), lac, self.fake_status_cb) + with self.assertRaises(morphlib.remoteartifactcache.GetError): + rac.get_artifacts( + chunk.artifacts.values(), lac, self.fake_status_cb) def test_escapes_pluses_in_request_urls(self): rac = morphlib.remoteartifactcache.RemoteArtifactCache( diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py index 004ba86e..71115f0d 100644 --- a/morphlib/remoterepocache.py +++ b/morphlib/remoterepocache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,10 +17,11 @@ import cliapp import json import logging -import urllib2 import urlparse import urllib +import requests + class ResolveRefError(cliapp.AppException): @@ -63,9 +64,9 @@ class RemoteRepoCache(object): repo_url = self._resolver.pull_url(repo_name) try: return self._cat_file_for_repo_url(repo_url, ref, filename) - except urllib2.HTTPError as e: + except requests.exceptions.HTTPError as e: logging.error('Caught exception: %s' % str(e)) - if e.code == 404: + if e.response.status_code == 404: raise CatFileError(repo_name, ref, filename) raise # pragma: no cover @@ -102,5 +103,10 @@ class RemoteRepoCache(object): if not server_url.endswith('/'): server_url += '/' url = urlparse.urljoin(server_url, '/1.0/%s' % path) - handle = urllib2.urlopen(url) - return handle.read() + + response = requests.get(url) + + if response.ok: + return response.text + else: + response.raise_for_status() diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py index ef81506f..d4f7d494 100644 --- a/morphlib/remoterepocache_tests.py +++ b/morphlib/remoterepocache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,7 +16,8 @@ import json import unittest -import urllib2 + +import requests import morphlib @@ -30,8 +31,9 @@ class RemoteRepoCacheTests(unittest.TestCase): try: return self.files[repo_url][sha1][filename] except KeyError: - raise urllib2.HTTPError(url='', code=404, msg='Not found', - hdrs={}, fp=None) + response = requests.Response() + response.status_code = 404 + raise requests.exceptions.HTTPError(response=response) def _ls_tree_for_repo_url(self, repo_url, sha1): return json.dumps({ |