diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-06-03 16:11:42 +0000 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-06-03 16:16:06 +0000 |
commit | 92547c0203a5b676020dd1f6816022454607c7b4 (patch) | |
tree | f8feb07625bf60db274a0cfa7cceb48e483e3082 /morphlib | |
parent | bfcae5cb0369c837b9acd8005a2f45496a18a81d (diff) | |
download | morph-92547c0203a5b676020dd1f6816022454607c7b4.tar.gz |
Ensure that transferring an artifact from the remote cache is atomic
Artifacts can have multiple parts; while this may not be an ideal
design, changing the format of artifacts has implications for backwards
compatibility.
We should transfer all parts at once and delete them all if we encounter
any errors, to reduce the change of getting the local artifact cache
into an inconsistent state.
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/buildcommand.py | 44 |
1 files changed, 29 insertions, 15 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 5e080a41..222b7127 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -387,27 +387,41 @@ class BuildCommand(object): def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' - def copy(remote, local): - shutil.copyfileobj(remote, local) - remote.close() - local.close() + def fetch_files(to_fetch): + '''Fetch a set of files atomically. + + If an error occurs during the transfer of any files, all downloaded + data is deleted, to ensure integrity of the local cache. + + ''' + try: + for remote, local in to_fetch: + shutil.copyfileobj(remote, local) + for remote, local in to_fetch: + remote.close() + local.close() + except BaseException: + for remote, local in to_fetch: + local.abort() + raise for artifact in artifacts: + to_fetch = [] if not self.lac.has(artifact): - self.app.status(msg='Fetching to local cache: ' - 'artifact %(name)s', - name=artifact.name) - rac_file = self.rac.get(artifact) - lac_file = self.lac.put(artifact) - copy(rac_file, lac_file) + to_fetch.append((self.rac.get(artifact), + self.lac.put(artifact))) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): - self.app.status(msg='Fetching to local cache: ' - 'artifact metadata %(name)s', - name=artifact.name) - copy(self.rac.get_artifact_metadata(artifact, 'meta'), - self.lac.put_artifact_metadata(artifact, 'meta')) + to_fetch.append(( + self.rac.get_artifact_metadata(artifact, 'meta'), + self.lac.put_artifact_metadata(artifact, 'meta'))) + + if len(to_fetch) > 0: + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, extra_path=[]): |