From 3d877091c9971ee5c5bd902bcdbf73fd8b08e001 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 10 Mar 2015 10:12:25 +0000 Subject: Make distbuild use an ArtifactReference not an Artifact internally when building We no longer serialise entire artifacts, so the output of deserialise_artifact is an ArtifactReference. This commit changes stuff in distbuild to know how to deal with that rather than an Artifact. Change-Id: I79b40d041700a85c25980e3bd70cd34dedd2a113 --- distbuild/build_controller.py | 29 ++++++++++++++--------------- distbuild/worker_build_scheduler.py | 29 ++++++++++++++++------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index 3971fe68..35b231f9 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -113,7 +113,7 @@ class _Abort(object): def build_step_name(artifact): '''Return user-comprehensible name for a given artifact.''' - return artifact.source.name + return artifact.source_name def map_build_graph(artifact, callback, components=[]): @@ -135,7 +135,7 @@ def map_build_graph(artifact, callback, components=[]): a = queue.pop() if a not in done: result.append(callback(a)) - queue.extend(a.source.dependencies) + queue.extend(a.dependencies) done.add(a) if a in components: mapped_components.append(a) @@ -145,8 +145,7 @@ def map_build_graph(artifact, callback, components=[]): def find_artifacts(components, artifact): found = [] for a in artifact.walk(): - name = a.source.morphology['name'] - if name in components: + if a.name in components: found.append(a) return found @@ -428,7 +427,7 @@ class BuildController(distbuild.StateMachine): def is_ready_to_build(artifact): return (artifact.state == UNBUILT and all(a.state == BUILT - for a in artifact.source.dependencies)) + for a in artifact.dependencies)) artifacts, _ = map_build_graph(self._artifact, lambda a: a, self._components) @@ -472,19 +471,19 @@ class BuildController(distbuild.StateMachine): logging.debug( 'Requesting worker-build of %s (%s)' % - (artifact.name, artifact.source.cache_key)) + (artifact.name, artifact.cache_key)) request = distbuild.WorkerBuildRequest(artifact, self._request['id']) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, request) artifact.state = BUILDING - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Chunk artifacts are not built independently # so when we're building any chunk artifact # we're also building all the chunk artifacts # in this source for a in ready: - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILDING def _maybe_notify_initiator_disconnected(self, event_source, event): @@ -588,7 +587,7 @@ class BuildController(distbuild.StateMachine): def _find_artifact(self, cache_key): artifacts, _ = map_build_graph(self._artifact, lambda a: a, self._components) - wanted = [a for a in artifacts if a.source.cache_key == cache_key] + wanted = [a for a in artifacts if a.cache_key == cache_key] if wanted: return wanted[0] else: @@ -614,10 +613,10 @@ class BuildController(distbuild.StateMachine): artifact.state = BUILT def set_state(a): - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILT - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Building a single chunk artifact # yields all chunk artifacts for the given source # so we set the state of this source's artifacts @@ -677,14 +676,14 @@ class BuildController(distbuild.StateMachine): urls = [] for c in self._components: name = ('%s.%s.%s' % - (c.source.cache_key, - c.source.morphology['kind'], + (c.cache_key, + c.kind, c.name)) urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) if not self._components: name = ('%s.%s.%s' % - (self._artifact.source.cache_key, - self._artifact.source.morphology['kind'], + (self._artifact.cache_key, + self._artifact.kind, self._artifact.name)) urls.append('%s?filename=%s' % (baseurl, urllib.quote(name))) diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index 8b581172..07962e15 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -276,13 +276,13 @@ class WorkerBuildQueuer(distbuild.StateMachine): logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, - event.artifact.source.cache_key, job.who.name()) + event.artifact.cache_key, job.who.name()) else: logging.debug('Job created but not building yet ' '(waiting for a worker to become available): %s' % event.artifact.basename()) progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.source.cache_key) + event.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) else: @@ -293,7 +293,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._give_job(job) else: progress = WorkerBuildWaiting(event.initiator_id, - event.artifact.source.cache_key) + event.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _handle_cancel(self, event_source, event): @@ -513,15 +513,18 @@ class WorkerConnection(distbuild.StateMachine): '--build-log-on-stdout', job.artifact.name, ] + msg = distbuild.message('exec-request', id=job.id, argv=argv, - stdin_contents=distbuild.serialise_artifact(job.artifact), + stdin_contents=distbuild.serialise_artifact(job.artifact, + job.artifact.repo, + job.artifact.ref), ) self._jm.send(msg) started = WorkerBuildStepStarted(job.initiators, - job.artifact.source.cache_key, self.name()) + job.artifact.cache_key, self.name()) self.mainloop.queue_event(WorkerConnection, _JobStarted(job)) self.mainloop.queue_event(WorkerConnection, started) @@ -557,7 +560,7 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('WC: emitting: %s', repr(new)) self.mainloop.queue_event( WorkerConnection, - WorkerBuildOutput(new, job.artifact.source.cache_key)) + WorkerBuildOutput(new, job.artifact.cache_key)) def _handle_exec_response(self, msg, job): '''Handle completion of a job that the worker is or was running.''' @@ -570,7 +573,7 @@ class WorkerConnection(distbuild.StateMachine): if new['exit'] != 0: # Build failed. - new_event = WorkerBuildFailed(new, job.artifact.source.cache_key) + new_event = WorkerBuildFailed(new, job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(WorkerConnection, _JobFailed(job)) self.mainloop.queue_event(self, _BuildFailed()) @@ -596,10 +599,10 @@ class WorkerConnection(distbuild.StateMachine): logging.debug('Requesting shared artifact cache to get artifacts') job = self._current_job - kind = job.artifact.source.morphology['kind'] + kind = job.artifact.kind if kind == 'chunk': - source_artifacts = job.artifact.source.artifacts + source_artifacts = job.artifact.source_artifacts suffixes = ['%s.%s' % (kind, name) for name in source_artifacts] suffixes.append('build-log') @@ -620,7 +623,7 @@ class WorkerConnection(distbuild.StateMachine): '/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' % (urllib.quote(worker_host), self._worker_cache_server_port, - urllib.quote(job.artifact.source.cache_key), + urllib.quote(job.artifact.cache_key), suffixes)) msg = distbuild.message( @@ -631,7 +634,7 @@ class WorkerConnection(distbuild.StateMachine): self.mainloop.queue_event(distbuild.HelperRouter, req) progress = WorkerBuildCaching(job.initiators, - job.artifact.source.cache_key) + job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, progress) def _maybe_handle_helper_result(self, event_source, event): @@ -644,7 +647,7 @@ class WorkerConnection(distbuild.StateMachine): new_event = WorkerBuildFinished( self._current_job_exec_response, - self._current_job.artifact.source.cache_key) + self._current_job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _Cached()) else: @@ -663,7 +666,7 @@ class WorkerConnection(distbuild.StateMachine): new_event = WorkerBuildFailed( self._current_job_exec_response, - self._current_job.artifact.source.cache_key) + self._current_job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) self.mainloop.queue_event(self, _BuildFailed()) -- cgit v1.2.1