Add a timeout to remote artifact cache connections

This means that builds won't hang forever if network connectivity drops.
Instead, Morph will show an error to the user, then try to do a local
build of the current artifact. (This may still break if the local git
cache does not contain the necessary SHA1s).

2 files changed, 24 insertions, 14 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 000a84e5..3d7bffe0 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -290,17 +290,13 @@ class BuildCommand(object):
         try:
             self.rac.get_artifacts_for_source(
                 source, self.lac, status_cb=self.app.status)
+        except morphlib.remoteartifactcache.NotCachedError as e:
+            self.app.status(msg='Not found in remote cache.')
         except morphlib.remoteartifactcache.GetError as e:
             # It's important to not hide the error, as the problem may be
             # something unexpected like a loose network cable.
-            if e.http_error.response.status_code == 404:
-                error = 'not cached.'
-            else:
-                error = e.http_error
-
             self.app.status(
-                msg='Unable to fetch artifact from cache: %(error)s',
-                error=error)
+                msg='Unable to fetch artifact from cache: %(error)s', error=e)
 
     def cache_or_build_source(self, source, build_env):
         '''Make artifacts of the built source available in the local cache.
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index a4caa27f..e53c3985 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -16,19 +16,26 @@
 
 import cliapp
 import logging
+import socket
 import urllib
 import urlparse
 
 import requests
 
 
-class GetError(cliapp.AppException):
+TIMEOUT_SECONDS = 10
+
 
-    def __init__(self, cache, filename, http_error):
-        self.http_error = http_error
+class GetError(cliapp.AppException):
+    def __init__(self, cache, filename, exception):
+        self.exception = exception
         cliapp.AppException.__init__(
             self, 'Failed to get the file %s from the artifact cache %s: %s' %
-                  (filename, cache, http_error))
+                  (filename, cache, exception))
+
+
+class NotCachedError(GetError):
+    pass
 
 
 class RemoteArtifactCache(object):
@@ -47,7 +54,7 @@ class RemoteArtifactCache(object):
         url = self._request_url(filename)
         logging.debug('RemoteArtifactCache._has_file: url=%s' % url)
 
-        response = self.requests.head(url)
+        response = self.requests.head(url, timeout=TIMEOUT_SECONDS)
 
         if response.status_code == 404:
             return False
@@ -91,15 +98,22 @@ class RemoteArtifactCache(object):
         logging.debug('RemoteArtifactCache._fetch_file: url=%s' % remote_url)
 
         try:
-            response = self.requests.get(remote_url, stream=True)
+            response = self.requests.get(remote_url, stream=True,
+                                         timeout=TIMEOUT_SECONDS)
             response.raise_for_status()
             content_length = int(response.headers.get('content-length', 0))
             for i, chunk in enumerate(response.iter_content(chunk_size)):
                 local_file.write(chunk)
                 show_status((i+1) * chunk_size, content_length)
+        except (socket.timeout, requests.exceptions.Timeout) as e:
+            logging.debug(str(e))
+            raise GetError(self, remote_filename, e)
         except requests.exceptions.HTTPError as e:
             logging.debug(str(e))
-            if e.response.status_code != 404 or error_if_missing:
+            if e.response.status_code == 404:
+                if error_if_missing:
+                    raise NotCachedError()
+            else:
                 raise GetError(self, remote_filename, e)
 
     def _fetch_files(self, to_fetch, status_cb):