From e740613e722fe3ac7d9f1885de5a2365d1833227 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Fri, 27 Feb 2015 15:23:53 +0000 Subject: Use requests instead of urllib2 to fetch an artifact Also make it retry 3 times to fetch an artifact, and add some debugging information useful when deploying/fixing a distbuild network. --- morph-cache-server | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index a3c3c97..ee790b5 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -22,7 +22,7 @@ import json import logging import os import urllib -import urllib2 +import requests import shutil from bottle import Bottle, request, response, run, static_file @@ -67,19 +67,31 @@ class MorphCacheServer(cliapp.Application): def _fetch_artifact(self, url, filename): - in_fh = None + r = None try: - in_fh = urllib2.urlopen(url) - with open(filename, "w") as localtmp: - shutil.copyfileobj(in_fh, localtmp) - in_fh.close() + logging.debug('Attempting to fetch with retries %s' % url) + # Create a requests Session and Adapter to retry 3 times. + s = requests.Session() + a = requests.adapters.HTTPAdapter(max_retries=3) + s.mount('http://', a) + # Set stream=True so requests doesn't download the whole artifact + # into memory first + r = s.get(url, stream=True) + if r.status_code == 200: + with open(filename, "w") as localtmp: + for chunk in r.iter_content(1024): + localtmp.write(chunk) + else: + raise Exception("Status code = %s" % r.status_code ) + r.close() + except Exception, e: - if in_fh is not None: - in_fh.close() + if r is not None: + r.close() raise else: - if in_fh is not None: - in_fh.close() + if r is not None: + r.close() return os.stat(filename) def _fetch_artifacts(self, server, cacheid, artifacts): -- cgit v1.2.1