WIP: Rough-and-ready method of reporting progress in artifact downloads

Change-Id: I37d158ee8786dcd8774cde07e9385939f043e905

4 files changed, 49 insertions, 7 deletions

diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index b95e0e77..346ccf32 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -75,7 +75,8 @@ class BuildCommand(object):
 
         '''
         return morphlib.util.new_artifact_caches(
-            self.app.settings, status_cb=self.app.status)
+            self.app.settings, status_cb=self.app.status,
+            runcmd=self.app.runcmd, verbose=self.app.settings['verbose'])
 
     def new_repo_caches(self):
         return morphlib.util.new_repo_caches(self.app)
diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py
index 40f724ca..3cc4a0fe 100644
--- a/morphlib/ostreeartifactcache.py
+++ b/morphlib/ostreeartifactcache.py
@@ -31,11 +31,15 @@ from morphlib.artifactcachereference import ArtifactCacheReference
 class OSTreeArtifactCache(object):
     """Class to provide the artifact cache API using an OSTree repo."""
 
-    def __init__(self, cachedir, status_cb=None):
+    def __init__(self, cachedir, status_cb=None, runcmd=cliapp.runcmd,
+                 verbose=False):
         repo_dir = os.path.join(cachedir, 'repo')
         self.repo = morphlib.ostree.OSTreeRepo(repo_dir)
         self.cachedir = cachedir
+
         self.status_cb = status_cb
+        self.runcmd = runcmd
+        self.verbose = verbose
 
     def status(self, *args, **kwargs):
         if self.status_cb is not None:
@@ -44,19 +48,21 @@ class OSTreeArtifactCache(object):
     @contextlib.contextmanager
     def _get_file_from_remote(self, artifact, remote, metadata_name=None):
         if metadata_name:
-            handle = remote.get_artifact_metadata(artifact, metadata_name)
+            url = remote.artifact_metadata_url(artifact, metadata_name)
             self.status(
                 msg='Downloading %(name)s %(metadata_name)s as a file.',
                 chatty=True, name=artifact.name, metadata_name=metadata_name)
         else:
-            handle = remote.get(artifact)
+            url = remote.url(artifact)
             self.status(
                 msg='Downloading %(name)s as a tarball.', chatty=True,
                 name=artifact.name)
 
         try:
             temporary_download = tempfile.NamedTemporaryFile(dir=self.cachedir)
-            shutil.copyfileobj(handle, temporary_download)
+            morphlib.util.fetch_url(
+                url, temporary_download.name, runcmd=self.runcmd,
+                progress_on_stdout=self.verbose)
             yield temporary_download.name
         finally:
             temporary_download.close()
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index f5115cd6..8b34e626 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -87,6 +87,9 @@ class RemoteArtifactCache(object):
             log(str(e))
             raise GetError(self, artifact)
 
+    def url(self, artifact):
+        return self._request_url(artifact.basename())
+
     def get_artifact_metadata(self, artifact, name, log=logging.error):
         try:
             return self._get_file(artifact.metadata_basename(name))
@@ -94,6 +97,9 @@ class RemoteArtifactCache(object):
             log(str(e))
             raise GetArtifactMetadataError(self, artifact, name)
 
+    def artifact_metadata_url(self, artifact, name):
+        return self._request_url(artifact.metadata_basename(name))
+
     def get_source_metadata(self, source, cachekey, name):
         filename = '%s.%s' % (cachekey, name)
         try:
@@ -101,6 +107,10 @@ class RemoteArtifactCache(object):
         except urllib2.URLError:
             raise GetSourceMetadataError(self, source, cachekey, name)
 
+    def source_metadata_url(self, source, cachekey, name):
+        filename = '%s.%s' % (cachekey, name)
+        return self._request_url(filename)
+
     def _has_file(self, filename):  # pragma: no cover
         url = self._request_url(filename)
         logging.debug('RemoteArtifactCache._has_file: url=%s' % url)
diff --git a/morphlib/util.py b/morphlib/util.py
index 7b610001..e1bb1ebe 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -18,8 +18,10 @@ import os
 import pipes
 import re
 import subprocess
+import sys
 import textwrap
 
+import cliapp
 import fs.osfs
 
 import morphlib
@@ -119,7 +121,8 @@ def get_git_resolve_cache_server(settings): # pragma: no cover
     return None
 
 
-def new_artifact_caches(settings, status_cb=None):  # pragma: no cover
+def new_artifact_caches(settings, status_cb=None, runcmd=cliapp.runcmd,
+                        verbose=False):  # pragma: no cover
     '''Create new objects for local and remote artifact caches.
 
     This includes creating the directories on disk, if missing.
@@ -135,7 +138,7 @@ def new_artifact_caches(settings, status_cb=None):  # pragma: no cover
     #        fs.osfs.OSFS(artifact_cachedir))
 
     lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(
-        artifact_cachedir, status_cb=status_cb)
+        artifact_cachedir, status_cb=status_cb, runcmd=runcmd, verbose=verbose)
 
     rac_url = get_artifact_cache_server(settings)
     rac = None
@@ -679,3 +682,25 @@ def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover
 
         os.fchown(f.fileno(), 0, 0)
         os.fchmod(f.fileno(), 0644)
+
+
+def fetch_url(url, target_path, runcmd=cliapp.runcmd,
+              progress_on_stdout=False):  # pragma: no cover
+    '''Fetch a remote HTTP URL to the local machine.
+
+    Unlike shutil.copyfileobj(), this method makes it easy to show progress
+    info to the user, which is important when downloading large files like
+    build artifacts.
+
+    '''
+    if progress_on_stdout:
+        verbosity_flags = []
+        kwargs = dict(stderr=sys.stdout)
+    else:
+        verbosity_flags = ['--quiet']
+        kwargs = dict()
+
+    def wget_command():
+        return ['wget'] + verbosity_flags + ['-O', target_path, url]
+
+    runcmd(wget_command(), **kwargs)