summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2014-06-03 16:11:42 +0000
committerSam Thursfield <sam.thursfield@codethink.co.uk>2014-06-03 16:16:06 +0000
commit92547c0203a5b676020dd1f6816022454607c7b4 (patch)
treef8feb07625bf60db274a0cfa7cceb48e483e3082 /morphlib
parentbfcae5cb0369c837b9acd8005a2f45496a18a81d (diff)
downloadmorph-92547c0203a5b676020dd1f6816022454607c7b4.tar.gz
Ensure that transferring an artifact from the remote cache is atomic
Artifacts can have multiple parts; while this may not be an ideal design, changing the format of artifacts has implications for backwards compatibility. We should transfer all parts at once and delete them all if we encounter any errors, to reduce the change of getting the local artifact cache into an inconsistent state.
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/buildcommand.py44
1 files changed, 29 insertions, 15 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 5e080a41..222b7127 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -387,27 +387,41 @@ class BuildCommand(object):
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
- def copy(remote, local):
- shutil.copyfileobj(remote, local)
- remote.close()
- local.close()
+ def fetch_files(to_fetch):
+ '''Fetch a set of files atomically.
+
+ If an error occurs during the transfer of any files, all downloaded
+ data is deleted, to ensure integrity of the local cache.
+
+ '''
+ try:
+ for remote, local in to_fetch:
+ shutil.copyfileobj(remote, local)
+ for remote, local in to_fetch:
+ remote.close()
+ local.close()
+ except BaseException:
+ for remote, local in to_fetch:
+ local.abort()
+ raise
for artifact in artifacts:
+ to_fetch = []
if not self.lac.has(artifact):
- self.app.status(msg='Fetching to local cache: '
- 'artifact %(name)s',
- name=artifact.name)
- rac_file = self.rac.get(artifact)
- lac_file = self.lac.put(artifact)
- copy(rac_file, lac_file)
+ to_fetch.append((self.rac.get(artifact),
+ self.lac.put(artifact)))
if artifact.source.morphology.needs_artifact_metadata_cached:
if not self.lac.has_artifact_metadata(artifact, 'meta'):
- self.app.status(msg='Fetching to local cache: '
- 'artifact metadata %(name)s',
- name=artifact.name)
- copy(self.rac.get_artifact_metadata(artifact, 'meta'),
- self.lac.put_artifact_metadata(artifact, 'meta'))
+ to_fetch.append((
+ self.rac.get_artifact_metadata(artifact, 'meta'),
+ self.lac.put_artifact_metadata(artifact, 'meta')))
+
+ if len(to_fetch) > 0:
+ self.app.status(
+ msg='Fetching to local cache: artifact %(name)s',
+ name=artifact.name)
+ fetch_files(to_fetch)
def create_staging_area(self, build_env, use_chroot=True, extra_env={},
extra_path=[]):