summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
authorRichard Ipsum <richardipsum@fastmail.co.uk>2015-07-05 12:15:55 +0100
committerRichard Maw <richard.maw@codethink.co.uk>2015-09-23 13:43:18 +0000
commit46511c257999b1c5bdfa02a1ae7d04ee26ffee8f (patch)
tree51537c6e85887dee7a93128521efc2fa6f12bcb9 /morphlib
parent4549b28282bb4c2591ccd24db5a29970a3e907f2 (diff)
downloadmorph-46511c257999b1c5bdfa02a1ae7d04ee26ffee8f.tar.gz
Display progress bar when fetching to local cache
Looks like, 2015-07-05 16:08:10 [Build 1/304] [stage1-binutils] Fetching to local cache: artifact stage1-binutils-misc stage1-binutils-misc[##################### ] 51.9/73.0 MB Change-Id: Ib10f1cfaa0c1df80ae605ecfeb5b706c8d46c4a4
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/buildcommand.py34
-rw-r--r--morphlib/util.py44
2 files changed, 75 insertions, 3 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index bb354b2f..8b728b05 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -1,4 +1,6 @@
+# -*- coding: utf-8 -*-
# Copyright (C) 2011-2015 Codethink Limited
+# Copyright © 2015 Richard Ipsum
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -400,16 +402,42 @@ class BuildCommand(object):
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
- def fetch_files(to_fetch):
+ def do_fetch(name, remote, local):
+ meta = remote.info()
+ content_len = int(meta.getheaders('Content-Length')[0])
+ logging.debug('Artifact content length: %s', content_len)
+
+ if content_len < 1024:
+ report_progress = lambda count: bar.show(count)
+ expected_size = content_len
+ unit = 'bytes'
+ elif content_len >= 1024 and content_len < 1024 ** 2:
+ report_progress = lambda count: bar.show(count / float(1024))
+ expected_size = content_len / float(1024)
+ unit = 'KB'
+ else:
+ report_progress = lambda count: bar.show(count
+ / float((1024 ** 2)))
+ expected_size = content_len / float((1024 ** 2))
+ unit = 'MB'
+
+ bar = morphlib.util.ProgressBar(name,
+ expected_size, unit)
+
+ morphlib.util.copyfileobj(remote, local,
+ callback=report_progress)
+
+ def fetch_files(name, to_fetch):
'''Fetch a set of files atomically.
If an error occurs during the transfer of any files, all downloaded
data is deleted, to ensure integrity of the local cache.
'''
+
try:
for remote, local in to_fetch:
- shutil.copyfileobj(remote, local)
+ do_fetch(name, remote, local)
except BaseException:
for remote, local in to_fetch:
local.abort()
@@ -439,7 +467,7 @@ class BuildCommand(object):
self.app.status(
msg='Fetching to local cache: artifact %(name)s',
name=artifact.name)
- fetch_files(to_fetch)
+ fetch_files(artifact.name, to_fetch)
def create_staging_area(self, source, build_env, use_chroot=True,
extra_env={}, extra_path=[]):
diff --git a/morphlib/util.py b/morphlib/util.py
index a92b7f37..284fe305 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -1,4 +1,6 @@
+# -*- coding: utf-8 -*-
# Copyright (C) 2011-2015 Codethink Limited
+# Copyright © 2015 Richard Ipsum
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -731,3 +733,45 @@ def temp_dir(*args, **kwargs): #pragma: no cover
else:
if cleanup_on_success:
shutil.rmtree(td, ignore_errors=True)
+
+def copyfileobj(fsrc, fdst, length=16*1024,
+ callback=lambda x: None): #pragma: no cover
+ ''' This is similar to shutil.copyfileobj
+ except this can be passed a callback to monitor copy progress
+ '''
+
+ count = 0
+
+ while True:
+ buf = fsrc.read(length)
+ if buf == '':
+ break
+
+ fdst.write(buf)
+ count += len(buf)
+ callback(count)
+
+
+class ProgressBar(object):
+ ''' A very simple progress bar '''
+
+ TEMPLATE = '%s[%s%s] %.1f/%.1f %s\r'
+
+ def __init__(self, label, expected_size, unit, width=30,
+ progress_char='#', empty_char=' '): #pragma: no cover
+ self._label = label
+ self._expected_size = expected_size
+ self._width = width
+ self._unit = unit
+ self._block_width = expected_size / float(width);
+ self._progress_char = progress_char
+ self._empty_char = empty_char
+
+ def show(self, progress): #pragma: no cover
+ blocks = int(progress / self._block_width);
+
+ s = self.TEMPLATE % (self._label, self._progress_char * blocks,
+ self._empty_char * (self._width - blocks),
+ progress, self._expected_size, self._unit)
+ sys.stderr.write(s)
+ sys.stderr.flush()