diff options
-rw-r--r-- | morphlib/buildcommand.py | 3 | ||||
-rw-r--r-- | morphlib/ostreeartifactcache.py | 14 | ||||
-rw-r--r-- | morphlib/remoteartifactcache.py | 10 | ||||
-rw-r--r-- | morphlib/util.py | 29 |
4 files changed, 49 insertions, 7 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index b95e0e77..346ccf32 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -75,7 +75,8 @@ class BuildCommand(object): ''' return morphlib.util.new_artifact_caches( - self.app.settings, status_cb=self.app.status) + self.app.settings, status_cb=self.app.status, + runcmd=self.app.runcmd, verbose=self.app.settings['verbose']) def new_repo_caches(self): return morphlib.util.new_repo_caches(self.app) diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py index 40f724ca..3cc4a0fe 100644 --- a/morphlib/ostreeartifactcache.py +++ b/morphlib/ostreeartifactcache.py @@ -31,11 +31,15 @@ from morphlib.artifactcachereference import ArtifactCacheReference class OSTreeArtifactCache(object): """Class to provide the artifact cache API using an OSTree repo.""" - def __init__(self, cachedir, status_cb=None): + def __init__(self, cachedir, status_cb=None, runcmd=cliapp.runcmd, + verbose=False): repo_dir = os.path.join(cachedir, 'repo') self.repo = morphlib.ostree.OSTreeRepo(repo_dir) self.cachedir = cachedir + self.status_cb = status_cb + self.runcmd = runcmd + self.verbose = verbose def status(self, *args, **kwargs): if self.status_cb is not None: @@ -44,19 +48,21 @@ class OSTreeArtifactCache(object): @contextlib.contextmanager def _get_file_from_remote(self, artifact, remote, metadata_name=None): if metadata_name: - handle = remote.get_artifact_metadata(artifact, metadata_name) + url = remote.artifact_metadata_url(artifact, metadata_name) self.status( msg='Downloading %(name)s %(metadata_name)s as a file.', chatty=True, name=artifact.name, metadata_name=metadata_name) else: - handle = remote.get(artifact) + url = remote.url(artifact) self.status( msg='Downloading %(name)s as a tarball.', chatty=True, name=artifact.name) try: temporary_download = tempfile.NamedTemporaryFile(dir=self.cachedir) - shutil.copyfileobj(handle, temporary_download) + morphlib.util.fetch_url( + url, temporary_download.name, runcmd=self.runcmd, + progress_on_stdout=self.verbose) yield temporary_download.name finally: temporary_download.close() diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index f5115cd6..8b34e626 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -87,6 +87,9 @@ class RemoteArtifactCache(object): log(str(e)) raise GetError(self, artifact) + def url(self, artifact): + return self._request_url(artifact.basename()) + def get_artifact_metadata(self, artifact, name, log=logging.error): try: return self._get_file(artifact.metadata_basename(name)) @@ -94,6 +97,9 @@ class RemoteArtifactCache(object): log(str(e)) raise GetArtifactMetadataError(self, artifact, name) + def artifact_metadata_url(self, artifact, name): + return self._request_url(artifact.metadata_basename(name)) + def get_source_metadata(self, source, cachekey, name): filename = '%s.%s' % (cachekey, name) try: @@ -101,6 +107,10 @@ class RemoteArtifactCache(object): except urllib2.URLError: raise GetSourceMetadataError(self, source, cachekey, name) + def source_metadata_url(self, source, cachekey, name): + filename = '%s.%s' % (cachekey, name) + return self._request_url(filename) + def _has_file(self, filename): # pragma: no cover url = self._request_url(filename) logging.debug('RemoteArtifactCache._has_file: url=%s' % url) diff --git a/morphlib/util.py b/morphlib/util.py index 7b610001..e1bb1ebe 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -18,8 +18,10 @@ import os import pipes import re import subprocess +import sys import textwrap +import cliapp import fs.osfs import morphlib @@ -119,7 +121,8 @@ def get_git_resolve_cache_server(settings): # pragma: no cover return None -def new_artifact_caches(settings, status_cb=None): # pragma: no cover +def new_artifact_caches(settings, status_cb=None, runcmd=cliapp.runcmd, + verbose=False): # pragma: no cover '''Create new objects for local and remote artifact caches. This includes creating the directories on disk, if missing. @@ -135,7 +138,7 @@ def new_artifact_caches(settings, status_cb=None): # pragma: no cover # fs.osfs.OSFS(artifact_cachedir)) lac = morphlib.ostreeartifactcache.OSTreeArtifactCache( - artifact_cachedir, status_cb=status_cb) + artifact_cachedir, status_cb=status_cb, runcmd=runcmd, verbose=verbose) rac_url = get_artifact_cache_server(settings) rac = None @@ -679,3 +682,25 @@ def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover os.fchown(f.fileno(), 0, 0) os.fchmod(f.fileno(), 0644) + + +def fetch_url(url, target_path, runcmd=cliapp.runcmd, + progress_on_stdout=False): # pragma: no cover + '''Fetch a remote HTTP URL to the local machine. + + Unlike shutil.copyfileobj(), this method makes it easy to show progress + info to the user, which is important when downloading large files like + build artifacts. + + ''' + if progress_on_stdout: + verbosity_flags = [] + kwargs = dict(stderr=sys.stdout) + else: + verbosity_flags = ['--quiet'] + kwargs = dict() + + def wget_command(): + return ['wget'] + verbosity_flags + ['-O', target_path, url] + + runcmd(wget_command(), **kwargs) |