diff options
-rwxr-xr-x | morph-cache-server | 32 |
1 files changed, 22 insertions, 10 deletions
diff --git a/morph-cache-server b/morph-cache-server index a3c3c97..ee790b5 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -22,7 +22,7 @@ import json import logging import os import urllib -import urllib2 +import requests import shutil from bottle import Bottle, request, response, run, static_file @@ -67,19 +67,31 @@ class MorphCacheServer(cliapp.Application): def _fetch_artifact(self, url, filename): - in_fh = None + r = None try: - in_fh = urllib2.urlopen(url) - with open(filename, "w") as localtmp: - shutil.copyfileobj(in_fh, localtmp) - in_fh.close() + logging.debug('Attempting to fetch with retries %s' % url) + # Create a requests Session and Adapter to retry 3 times. + s = requests.Session() + a = requests.adapters.HTTPAdapter(max_retries=3) + s.mount('http://', a) + # Set stream=True so requests doesn't download the whole artifact + # into memory first + r = s.get(url, stream=True) + if r.status_code == 200: + with open(filename, "w") as localtmp: + for chunk in r.iter_content(1024): + localtmp.write(chunk) + else: + raise Exception("Status code = %s" % r.status_code ) + r.close() + except Exception, e: - if in_fh is not None: - in_fh.close() + if r is not None: + r.close() raise else: - if in_fh is not None: - in_fh.close() + if r is not None: + r.close() return os.stat(filename) def _fetch_artifacts(self, server, cacheid, artifacts): |