Use requests instead of urllib2 to fetch an artifactbaserock/pedroalvarez/use-requests

Also make it retry 3 times to fetch an artifact, and add some debugging
information useful when deploying/fixing a distbuild network.

1 files changed, 22 insertions, 10 deletions

diff --git a/morph-cache-server b/morph-cache-server
index a3c3c97..ee790b5 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -22,7 +22,7 @@ import json
 import logging
 import os
 import urllib
-import urllib2
+import requests
 import shutil
 
 from bottle import Bottle, request, response, run, static_file
@@ -67,19 +67,31 @@ class MorphCacheServer(cliapp.Application):
 
 
     def _fetch_artifact(self, url, filename):
-        in_fh = None
+        r = None
         try:
-            in_fh = urllib2.urlopen(url)
-            with open(filename, "w") as localtmp:
-                shutil.copyfileobj(in_fh, localtmp)
-            in_fh.close()
+            logging.debug('Attempting to fetch with retries %s' % url)
+            # Create a requests Session and Adapter to retry 3 times.
+            s = requests.Session()
+            a = requests.adapters.HTTPAdapter(max_retries=3)
+            s.mount('http://', a)
+            # Set stream=True so requests doesn't download the whole artifact
+            # into memory first
+            r = s.get(url, stream=True)
+            if r.status_code == 200:
+                with open(filename, "w") as localtmp:
+                    for chunk in r.iter_content(1024):
+                        localtmp.write(chunk)
+            else:
+                raise Exception("Status code = %s" % r.status_code )
+            r.close()
+
         except Exception, e:
-            if in_fh is not None:
-                in_fh.close()
+            if r is not None:
+                r.close()
             raise
         else:
-            if in_fh is not None:
-                in_fh.close()
+            if r is not None:
+                r.close()
         return os.stat(filename)
 
     def _fetch_artifacts(self, server, cacheid, artifacts):