From bcd48e0026af2ab7befbdee9735d27474bc4c57c Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 10 Mar 2015 10:12:25 +0000 Subject: Make distbuild use an ArtifactReference not an Artifact internally when building We no longer serialise entire artifacts, so the output of deserialise_artifact is an ArtifactReference. This commit changes stuff in distbuild to know how to deal with that rather than an Artifact. --- distbuild/build_controller.py | 22 +++++++++++----------- distbuild/worker_build_scheduler.py | 29 ++++++++++++++++------------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index d6f3398f..3a0dd9b0 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -113,7 +113,7 @@ class _Abort(object): def build_step_name(artifact): '''Return user-comprehensible name for a given artifact.''' - return artifact.source.name + return artifact.source_name def map_build_graph(artifact, callback): @@ -124,7 +124,7 @@ def map_build_graph(artifact, callback): a = queue.pop() if a not in done: result.append(callback(a)) - queue.extend(a.source.dependencies) + queue.extend(a.dependencies) done.add(a) return result @@ -373,7 +373,7 @@ class BuildController(distbuild.StateMachine): def is_ready_to_build(artifact): return (artifact.state == UNBUILT and all(a.state == BUILT - for a in artifact.source.dependencies)) + for a in artifact.dependencies)) return [a for a in map_build_graph(self._artifact, lambda a: a) @@ -409,19 +409,19 @@ class BuildController(distbuild.StateMachine): logging.debug( 'Requesting worker-build of %s (%s)' % - (artifact.name, artifact.source.cache_key)) + (artifact.name, artifact.cache_key)) request = distbuild.WorkerBuildRequest(artifact, self._request['id']) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, request) artifact.state = BUILDING - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Chunk artifacts are not built independently # so when we're building any chunk artifact # we're also building all the chunk artifacts # in this source for a in ready: - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILDING @@ -525,7 +525,7 @@ class BuildController(distbuild.StateMachine): def _find_artifact(self, cache_key): artifacts = map_build_graph(self._artifact, lambda a: a) - wanted = [a for a in artifacts if a.source.cache_key == cache_key] + wanted = [a for a in artifacts if a.cache_key == cache_key] if wanted: return wanted[0] else: @@ -551,10 +551,10 @@ class BuildController(distbuild.StateMachine): artifact.state = BUILT def set_state(a): - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILT - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Building a single chunk artifact # yields all chunk artifacts for the given source # so we set the state of this source's artifacts @@ -611,8 +611,8 @@ class BuildController(distbuild.StateMachine): baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') filename = ('%s.%s.%s' % - (self._artifact.source.cache_key, - self._artifact.source.morphology['kind'], + (self._artifact.cache_key, + self._artifact.kind, self._artifact.name)) url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) finished = BuildFinished(self._request['id'], [url]) diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index e58059b2..d00b0290 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -270,13 +270,13 @@ class WorkerBuildQueuer(distbuild.StateMachine): logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, - event.artifact.source.cache_key, job.who.name()) + event.artifact.cache_key, job.who.name()) else: logging.debug('Job created but not building yet ' '(waiting for a worker to become available): %s' % event.artifact.basename()) progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.source.cache_key) + event.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) else: @@ -287,7 +287,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._give_job(job) else: progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.source.cache_key) + event.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _handle_cancel(self, event_source, event): @@ -506,10 +506,13 @@ class WorkerConnection(distbuild.StateMachine): '--build-log-on-stdout', job.artifact.name, ] + msg = distbuild.message('exec-request', id=job.id, argv=argv, - stdin_contents=distbuild.serialise_artifact(job.artifact), + stdin_contents=distbuild.serialise_artifact(job.artifact, + job.artifact.repo, + job.artifact.ref), ) self._jm.send(msg) @@ -518,7 +521,7 @@ class WorkerConnection(distbuild.StateMachine): % (self._worker_name, msg)) started = WorkerBuildStepStarted(job.initiators, - job.artifact.source.cache_key, self.name()) + job.artifact.cache_key, self.name()) self.mainloop.queue_event(WorkerConnection, _JobStarted(job)) self.mainloop.queue_event(WorkerConnection, started) @@ -554,7 +557,7 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('WC: emitting: %s', repr(new)) self.mainloop.queue_event( WorkerConnection, - WorkerBuildOutput(new, job.artifact.source.cache_key)) + WorkerBuildOutput(new, job.artifact.cache_key)) def _handle_exec_response(self, msg, job): '''Handle completion of a job that the worker is or was running.''' @@ -567,7 +570,7 @@ class WorkerConnection(distbuild.StateMachine): if new['exit'] != 0: # Build failed. - new_event = WorkerBuildFailed(new, job.artifact.source.cache_key) + new_event = WorkerBuildFailed(new, job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(WorkerConnection, _JobFailed(job)) self.mainloop.queue_event(self, _BuildFailed()) @@ -593,10 +596,10 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('Requesting shared artifact cache to get artifacts') job = self._current_job - kind = job.artifact.source.morphology['kind'] + kind = job.artifact.kind if kind == 'chunk': - source_artifacts = job.artifact.source.artifacts + source_artifacts = job.artifact.source_artifacts suffixes = ['%s.%s' % (kind, name) for name in source_artifacts] suffixes.append('build-log') @@ -617,7 +620,7 @@ class WorkerConnection(distbuild.StateMachine): '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' % (urllib.quote(worker_host), self._worker_cache_server_port, - urllib.quote(job.artifact.source.cache_key), + urllib.quote(job.artifact.cache_key), suffixes)) msg = distbuild.message( @@ -628,7 +631,7 @@ class WorkerConnection(distbuild.StateMachine): self.mainloop.queue_event(distbuild.HelperRouter, req) progress = WorkerBuildCaching(job.initiators, - job.artifact.source.cache_key) + job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _maybe_handle_helper_result(self, event_source, event): @@ -641,7 +644,7 @@ class WorkerConnection(distbuild.StateMachine): new_event = WorkerBuildFinished( self._current_job_exec_response, - self._current_job.artifact.source.cache_key) + self._current_job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _Cached()) else: @@ -660,7 +663,7 @@ class WorkerConnection(distbuild.StateMachine): new_event = WorkerBuildFailed( self._current_job_exec_response, - self._current_job.artifact.source.cache_key) + self._current_job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _BuildFailed()) -- cgit v1.2.1