summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2014-06-10 12:18:38 +0100
committerSam Thursfield <sam.thursfield@codethink.co.uk>2014-06-10 12:19:29 +0100
commitbe6870d1145634401f912e7f5623da27b20180cd (patch)
tree801b4afcbd39c8c4e22e9d54ba0e9f6c29c44d50
parent623e77cf0a9e3b018195490870e3a2e1f003d31d (diff)
parent29c836174baff2b6fa02d8ce7109ff656c4d4834 (diff)
downloadmorph-be6870d1145634401f912e7f5623da27b20180cd.tar.gz
Merge branch 'sam/remote-artifact-cache-failure'
Reviewed-By: Richard Ipsum <richard.ipsum@codethink.co.uk> Reviewed-By: Richard Maw <richard.maw@codethink.co.uk>
-rw-r--r--morphlib/buildcommand.py85
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py14
-rw-r--r--morphlib/remoteartifactcache.py11
3 files changed, 60 insertions, 50 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 7ad7909d..5abef54c 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -274,31 +274,31 @@ class BuildCommand(object):
'name': a.name,
})
- self.app.status(msg='Checking if %(kind)s needs '
- 'building %(sha1)s',
- kind=a.source.morphology['kind'],
- sha1=a.source.sha1[:7])
-
- if self.is_built(a):
- self.cache_artifacts_locally([a])
- self.app.status(
- msg='The %(kind)s is cached at %(cache)s',
- kind=a.source.morphology['kind'],
- cache=os.path.basename(self.lac.artifact_filename(a))[:7])
- else:
- self.app.status(msg='Building %(kind)s %(name)s',
- name=a.name, kind=a.source.morphology['kind'])
- self.build_artifact(a, build_env)
+ self.cache_or_build_artifact(a, build_env)
self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
kind=a.source.morphology['kind'], name=a.name,
cachepath=self.lac.artifact_filename(a),
chatty=(a.source.morphology['kind'] != "system"))
+
self.app.status_prefix = old_prefix
- def is_built(self, artifact):
- '''Does either cache already have the artifact?'''
- return self.lac.has(artifact) or (self.rac and self.rac.has(artifact))
+ def cache_or_build_artifact(self, artifact, build_env):
+ '''Make the built artifact available in the local cache.
+
+ This can be done by retrieving from a remote artifact cache, or if
+ that doesn't work for some reason, by building the artifact locally.
+
+ '''
+ if self.rac is not None:
+ try:
+ self.cache_artifacts_locally([artifact])
+ except morphlib.remoteartifactcache.GetError:
+ # Error is logged by the RemoteArtifactCache object.
+ pass
+
+ if not self.lac.has(artifact):
+ self.build_artifact(artifact, build_env)
def build_artifact(self, artifact, build_env):
'''Build one artifact.
@@ -307,6 +307,10 @@ class BuildCommand(object):
in either the local or remote cache already.
'''
+ self.app.status(msg='Building %(kind)s %(name)s',
+ name=artifact.name,
+ kind=artifact.source.morphology['kind'])
+
self.get_sources(artifact)
deps = self.get_recursive_deps(artifact)
self.cache_artifacts_locally(deps)
@@ -389,27 +393,42 @@ class BuildCommand(object):
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
- def copy(remote, local):
- shutil.copyfileobj(remote, local)
- remote.close()
- local.close()
+ def fetch_files(to_fetch):
+ '''Fetch a set of files atomically.
+
+ If an error occurs during the transfer of any files, all downloaded
+ data is deleted, to ensure integrity of the local cache.
+
+ '''
+ try:
+ for remote, local in to_fetch:
+ shutil.copyfileobj(remote, local)
+ except BaseException:
+ for remote, local in to_fetch:
+ local.abort()
+ raise
+ else:
+ for remote, local in to_fetch:
+ remote.close()
+ local.close()
for artifact in artifacts:
+ to_fetch = []
if not self.lac.has(artifact):
- self.app.status(msg='Fetching to local cache: '
- 'artifact %(name)s',
- name=artifact.name)
- rac_file = self.rac.get(artifact)
- lac_file = self.lac.put(artifact)
- copy(rac_file, lac_file)
+ to_fetch.append((self.rac.get(artifact),
+ self.lac.put(artifact)))
if artifact.source.morphology.needs_artifact_metadata_cached:
if not self.lac.has_artifact_metadata(artifact, 'meta'):
- self.app.status(msg='Fetching to local cache: '
- 'artifact metadata %(name)s',
- name=artifact.name)
- copy(self.rac.get_artifact_metadata(artifact, 'meta'),
- self.lac.put_artifact_metadata(artifact, 'meta'))
+ to_fetch.append((
+ self.rac.get_artifact_metadata(artifact, 'meta'),
+ self.lac.put_artifact_metadata(artifact, 'meta')))
+
+ if len(to_fetch) > 0:
+ self.app.status(
+ msg='Fetching to local cache: artifact %(name)s',
+ name=artifact.name)
+ fetch_files(to_fetch)
def create_staging_area(self, build_env, use_chroot=True, extra_env={},
extra_path=[]):
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index ec0cfbcb..bfd0d047 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -299,18 +299,8 @@ class CrossBootstrapPlugin(cliapp.Plugin):
'Nothing to cross-compile. Only chunks built in \'bootstrap\' '
'mode can be cross-compiled.')
- # FIXME: merge with build-command's code
for i, a in enumerate(cross_chunks):
- if build_command.is_built(a):
- self.app.status(msg='The %(kind)s %(name)s is already built',
- kind=a.source.morphology['kind'],
- name=a.name)
- build_command.cache_artifacts_locally([a])
- else:
- self.app.status(msg='Cross-building %(kind)s %(name)s',
- kind=a.source.morphology['kind'],
- name=a.name)
- build_command.build_artifact(a, build_env)
+ build_command.cache_or_build_artifact(a, build_env)
for i, a in enumerate(native_chunks):
build_command.get_sources(a)
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index 9f6bf69e..0f8edce8 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2013 Codethink Limited
+# Copyright (C) 2012-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -33,18 +33,19 @@ class GetError(cliapp.AppException):
cliapp.AppException.__init__(
self, 'Failed to get the artifact %s with cache key %s '
'from the artifact cache %s' %
- (artifact, artifact.cache_key, cache))
+ (artifact.basename(), artifact.cache_key, cache))
-class GetArtifactMetadataError(cliapp.AppException):
+class GetArtifactMetadataError(GetError):
def __init__(self, cache, artifact, name):
cliapp.AppException.__init__(
self, 'Failed to get metadata %s for the artifact %s '
- 'from the artifact cache %s' % (name, artifact, cache))
+ 'from the artifact cache %s' %
+ (name, artifact.basename(), cache))
-class GetSourceMetadataError(cliapp.AppException):
+class GetSourceMetadataError(GetError):
def __init__(self, cache, source, cache_key, name):
cliapp.AppException.__init__(