diff options
author | Pedro Alvarez <pedro.alvarez@codethink.co.uk> | 2015-02-27 15:23:53 +0000 |
---|---|---|
committer | Pedro Alvarez <pedro.alvarez@codethink.co.uk> | 2015-02-27 16:43:45 +0000 |
commit | e740613e722fe3ac7d9f1885de5a2365d1833227 (patch) | |
tree | 2b8803263d695e08b7bf7b1171a39f736b12229a | |
parent | acefe33868585cf31cda53474a3004da42e00896 (diff) | |
download | morph-cache-server-baserock/pedroalvarez/use-requests.tar.gz |
Use requests instead of urllib2 to fetch an artifactbaserock/pedroalvarez/use-requests
Also make it retry 3 times to fetch an artifact, and add some debugging
information useful when deploying/fixing a distbuild network.
-rwxr-xr-x | morph-cache-server | 32 |
1 files changed, 22 insertions, 10 deletions
diff --git a/morph-cache-server b/morph-cache-server index a3c3c97..ee790b5 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -22,7 +22,7 @@ import json import logging import os import urllib -import urllib2 +import requests import shutil from bottle import Bottle, request, response, run, static_file @@ -67,19 +67,31 @@ class MorphCacheServer(cliapp.Application): def _fetch_artifact(self, url, filename): - in_fh = None + r = None try: - in_fh = urllib2.urlopen(url) - with open(filename, "w") as localtmp: - shutil.copyfileobj(in_fh, localtmp) - in_fh.close() + logging.debug('Attempting to fetch with retries %s' % url) + # Create a requests Session and Adapter to retry 3 times. + s = requests.Session() + a = requests.adapters.HTTPAdapter(max_retries=3) + s.mount('http://', a) + # Set stream=True so requests doesn't download the whole artifact + # into memory first + r = s.get(url, stream=True) + if r.status_code == 200: + with open(filename, "w") as localtmp: + for chunk in r.iter_content(1024): + localtmp.write(chunk) + else: + raise Exception("Status code = %s" % r.status_code ) + r.close() + except Exception, e: - if in_fh is not None: - in_fh.close() + if r is not None: + r.close() raise else: - if in_fh is not None: - in_fh.close() + if r is not None: + r.close() return os.stat(filename) def _fetch_artifacts(self, server, cacheid, artifacts): |