diff options
Diffstat (limited to 'morphlib/buildcommand.py')
| -rw-r--r-- | morphlib/buildcommand.py | 102 |
1 files changed, 67 insertions, 35 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 7ad7909d..f68046e3 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -274,31 +274,31 @@ class BuildCommand(object): 'name': a.name, }) - self.app.status(msg='Checking if %(kind)s needs ' - 'building %(sha1)s', - kind=a.source.morphology['kind'], - sha1=a.source.sha1[:7]) - - if self.is_built(a): - self.cache_artifacts_locally([a]) - self.app.status( - msg='The %(kind)s is cached at %(cache)s', - kind=a.source.morphology['kind'], - cache=os.path.basename(self.lac.artifact_filename(a))[:7]) - else: - self.app.status(msg='Building %(kind)s %(name)s', - name=a.name, kind=a.source.morphology['kind']) - self.build_artifact(a, build_env) + self.cache_or_build_artifact(a, build_env) self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', kind=a.source.morphology['kind'], name=a.name, cachepath=self.lac.artifact_filename(a), chatty=(a.source.morphology['kind'] != "system")) + self.app.status_prefix = old_prefix - def is_built(self, artifact): - '''Does either cache already have the artifact?''' - return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + def cache_or_build_artifact(self, artifact, build_env): + '''Make the built artifact available in the local cache. + + This can be done by retrieving from a remote artifact cache, or if + that doesn't work for some reason, by building the artifact locally. + + ''' + if self.rac is not None: + try: + self.cache_artifacts_locally([artifact]) + except morphlib.remoteartifactcache.GetError: + # Error is logged by the RemoteArtifactCache object. + pass + + if not self.lac.has(artifact): + self.build_artifact(artifact, build_env) def build_artifact(self, artifact, build_env): '''Build one artifact. @@ -307,6 +307,10 @@ class BuildCommand(object): in either the local or remote cache already. ''' + self.app.status(msg='Building %(kind)s %(name)s', + name=artifact.name, + kind=artifact.source.morphology['kind']) + self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) @@ -389,27 +393,46 @@ class BuildCommand(object): def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' - def copy(remote, local): - shutil.copyfileobj(remote, local) - remote.close() - local.close() + def fetch_files(to_fetch): + '''Fetch a set of files atomically. + + If an error occurs during the transfer of any files, all downloaded + data is deleted, to ensure integrity of the local cache. + + ''' + try: + for remote, local in to_fetch: + shutil.copyfileobj(remote, local) + except BaseException: + for remote, local in to_fetch: + local.abort() + raise + else: + for remote, local in to_fetch: + remote.close() + local.close() for artifact in artifacts: + # This block should fetch all artifact files in one go, using the + # 1.0/artifacts method of morph-cache-server. The code to do that + # needs bringing in from the distbuild.worker_build_connection + # module into morphlib.remoteartififactcache first. + to_fetch = [] if not self.lac.has(artifact): - self.app.status(msg='Fetching to local cache: ' - 'artifact %(name)s', - name=artifact.name) - rac_file = self.rac.get(artifact) - lac_file = self.lac.put(artifact) - copy(rac_file, lac_file) + to_fetch.append((self.rac.get(artifact), + self.lac.put(artifact))) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): - self.app.status(msg='Fetching to local cache: ' - 'artifact metadata %(name)s', - name=artifact.name) - copy(self.rac.get_artifact_metadata(artifact, 'meta'), - self.lac.put_artifact_metadata(artifact, 'meta')) + to_fetch.append(( + self.rac.get_artifact_metadata(artifact, 'meta'), + self.lac.put_artifact_metadata(artifact, 'meta'))) + + if len(to_fetch) > 0: + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, extra_path=[]): @@ -479,6 +502,9 @@ class BuildCommand(object): class InitiatorBuildCommand(BuildCommand): + RECONNECT_INTERVAL = 30 # seconds + MAX_RETRIES = 1 + def __init__(self, app, addr, port): self.app = app self.addr = addr @@ -501,7 +527,13 @@ class InitiatorBuildCommand(BuildCommand): self.app.status(msg='Starting distributed build') loop = distbuild.MainLoop() - cm = distbuild.ConnectionMachine( - self.addr, self.port, distbuild.Initiator, [self.app] + args) + cm = distbuild.InitiatorConnectionMachine(self.app, + self.addr, + self.port, + distbuild.Initiator, + [self.app] + args, + self.RECONNECT_INTERVAL, + self.MAX_RETRIES) + loop.add_state_machine(cm) loop.run() |