Whatever

4 files changed, 84 insertions, 28 deletions

diff --git a/morphlib/artifact.py b/morphlib/artifact.py
index ae1d0604..4a385281 100644
--- a/morphlib/artifact.py
+++ b/morphlib/artifact.py
@@ -62,3 +62,22 @@ class Artifact(object):
                 yield a
 
         return list(depth_first(self))
+
+
+def find_all_deps(artifacts):
+    '''Return all sources that are dependencies of a set of artifacts.
+
+    Deps are returned in order of 'nearest' to 'furthest' from the root
+    artifacts.
+
+    '''
+
+    deps = set()
+    ordered_deps = []
+
+    for artifact in artifacts:
+        for dep in artifact.walk():
+            if dep.source not in deps and dep not in artifacts:
+                deps.add(dep.source)
+                ordered_deps.append(dep.source)
+    return ordered_deps
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index d97d52fb..1fce2950 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -286,6 +286,21 @@ class BuildCommand(object):
 
         self.app.status_prefix = old_prefix
 
+    def maybe_cache_artifacts_locally(self, source):
+        try:
+            self.cache_artifacts_locally([source])
+        except morphlib.remoteartifactcache.GetError as e:
+            # It's important to not hide the error, as the problem may be
+            # something unexpected like a loose network cable.
+            if e.http_error.response.status_code == 404:
+                error = 'not cached.'
+            else:
+                error = e.http_error
+
+            self.app.status(
+                msg='Unable to fetch artifact from cache: %(error)s',
+                error=error)
+
     def cache_or_build_source(self, source, build_env):
         '''Make artifacts of the built source available in the local cache.
 
@@ -293,28 +308,19 @@ class BuildCommand(object):
         that doesn't work for some reason, by building the source locally.
 
         '''
-        if self.rac is not None:
-            try:
-                self.cache_artifacts_locally([source])
-            except morphlib.remoteartifactcache.GetError as e:
-                if e.http_error.response.status_code == 404:
-                    error = 'not present.'
-                else
-                    error = e.http_error
-
-                self.status(
-                    msg='Unable to fetch artifact from cache: %(error)s'
-                    error=error)
-                pass
-
-        artifacts = source.artifacts.values()
-        if any(not self.lac.has(artifact) for artifact in artifacts):
-            # If any of the artifacts are missing, build the whole thing again.
-            # We should only ever have all the artifacts or none of them, but
-            # builds needs to be robust.
+        def artifacts_available_locally(source):
+            # If any of the artifacts are missing, we consider the whole thing
+            # missing.
+            artifacts = source.artifacts.values()
+            return all(self.lac.has(artifact) for artifact in artifacts)
+
+        if not artifacts_available_locally(source) and self.rac:
+            self.maybe_cache_artifacts_locally(source)
+
+        if not artifacts_available_locally(source):
             self.build_source(source, build_env)
 
-        for a in artifacts:
+        for a in source.artifacts.values():
             self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
                             kind=source.morphology['kind'], name=a.name,
                             cachepath=self.lac.artifact_filename(a),
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index 1b48f8f4..b4036385 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -63,8 +63,23 @@ class RemoteArtifactCache(object):
     def __str__(self):  # pragma: no cover
         return self.server_url
 
-    def _fetch_file(self, remote_filename, local_file):
-        chunk_size = 1024 * 100
+    def _fetch_file(self, remote_filename, local_file, status_cb=None,
+                    error_if_missing=True):
+        chunk_size = 10 * 1024 * 1024
+
+        def show_status(downloaded, total):
+            if not status_cb:
+                return
+            downloaded = min(downloaded, total)
+            if total is None:
+                status_cb(msg='Fetched %(downloaded).02fMB',
+                          downloaded=downloaded / (1024 * 1024),
+                          chatty=True)
+            else:
+                status_cb(msg='Fetched %(downloaded).02fMB of %(total).02fMB',
+                          downloaded=downloaded / (1024 * 1024),
+                          total=total / (1024 * 1024),
+                          chatty=True)
 
         remote_url = self._request_url(remote_filename)
         logging.debug('RemoteArtifactCache._fetch_file: url=%s' % remote_url)
@@ -72,13 +87,16 @@ class RemoteArtifactCache(object):
         try:
             response = requests.get(remote_url, stream=True)
             response.raise_for_status()
-            for chunk in response.iter_content(chunk_size):
+            content_length = int(response.headers.get('content-length'))
+            for i, chunk in enumerate(response.iter_content(chunk_size)):
                 local_file.write(chunk)
+                show_status(i+1 * chunk_size, content_length)
         except requests.exceptions.HTTPError as e:
             logging.debug(str(e))
-            raise GetError(self, remote_filename, e)
+            if e.response.status_code != 404 or error_if_missing:
+                raise GetError(self, remote_filename, e)
 
-    def _fetch_files(self, to_fetch):
+    def _fetch_files(self, to_fetch, status_cb):
         '''Fetch a set of files atomically.
 
         If an error occurs during the transfer of any files, all downloaded
@@ -89,9 +107,19 @@ class RemoteArtifactCache(object):
         handles passed in to_fetch have a .abort() method.
 
         '''
+        def is_required(basename):
+            # This is a workaround for a historical bugs. Distbuild used to
+            # fail to transfer the source .meta and the .build-log file, so
+            # we need to cope if they are missing.
+            if basename.endswith('.build-log') or basename.endswith('.meta'):
+                return False
+            return True
+
         try:
             for remote_filename, local_file in to_fetch:
-                self._fetch_file(remote_filename, local_file)
+                self._fetch_file(
+                    remote_filename, local_file, status_cb=status_cb,
+                    error_if_missing=is_required(remote_filename))
         except BaseException:
             for _, local_file in to_fetch:
                 local_file.abort()
@@ -119,7 +147,7 @@ class RemoteArtifactCache(object):
                     msg='Fetching built artifacts of %(name)s',
                     name=source.name)
 
-            self._fetch_files(to_fetch)
+            self._fetch_files(to_fetch, status_cb=status_cb)
 
     def get_artifacts_for_sources(self, sources, lac, status_cb=None):
         '''Ensure artifacts for multiple sources are available locally.'''
diff --git a/morphlib/source.py b/morphlib/source.py
index 83f61dad..53563a43 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -67,6 +67,9 @@ class Source(object):
     def basename(self):
         return '%s.%s' % (self.cache_key, str(self.morphology['kind']))
 
+    def metadata_basename(self):
+        return '%s.meta' % (self.cache_key)
+
     def build_log_basename(self):
         return '%s.build-log' % (self.cache_key)
 
@@ -79,7 +82,7 @@ class Source(object):
         source available. Transfer all of them if you transfer any of them.
 
         '''
-        files = {self.basename()}
+        files = {self.metadata_basename()}
 
         if self.morphology['kind'] == 'chunk':
             files.add(self.build_log_basename())