From 46a8e0de14ce2cca4c93ec53648d7e01333bbf65 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 18 Feb 2015 11:23:56 +0000 Subject: distbuild: Remove the build-steps message This message was hundreds of kilobytes in size, as it contained a recursive list of dependencies for each artifact in the build graph. It was used in the initiator only to print this message: Build steps in total: 592 This message is now gone. The 'Need to build %d artifacts' build-progress message now indicates the total build steps instead: Need to build 300 artifacts, of 592 total This is a compatible change to the distbuild protocol: old initiators will continue to work as normal with new controllers that don't send the build-steps message. --- distbuild/__init__.py | 5 ++--- distbuild/build_controller.py | 20 ++++---------------- distbuild/initiator.py | 8 -------- distbuild/initiator_connection.py | 22 ---------------------- distbuild/protocol.py | 4 ---- 5 files changed, 6 insertions(+), 53 deletions(-) diff --git a/distbuild/__init__.py b/distbuild/__init__.py index 0bfae6e9..62a904d4 100644 --- a/distbuild/__init__.py +++ b/distbuild/__init__.py @@ -51,9 +51,8 @@ from worker_build_scheduler import (WorkerBuildQueuer, WorkerBuildFailed, WorkerBuildStepStarted) from build_controller import (BuildController, BuildFailed, BuildProgress, - BuildSteps, BuildStepStarted, - BuildStepAlreadyStarted, BuildOutput, - BuildStepFinished, BuildStepFailed, + BuildStepStarted, BuildStepAlreadyStarted, + BuildOutput, BuildStepFinished, BuildStepFailed, BuildFinished, BuildCancel, build_step_name, map_build_graph) from initiator import Initiator diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index 32da92f4..fd4b2105 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -71,13 +71,6 @@ class BuildProgress(object): self.message_text = message_text -class BuildSteps(object): - - def __init__(self, request_id, artifact): - self.id = request_id - self.artifact = artifact - - class BuildStepStarted(object): def __init__(self, request_id, step_name, worker_name): @@ -295,9 +288,6 @@ class BuildController(distbuild.StateMachine): progress = BuildProgress( self._request['id'], 'Finished computing build graph') self.mainloop.queue_event(BuildController, progress) - - build_steps = BuildSteps(self._request['id'], artifact) - self.mainloop.queue_event(BuildController, build_steps) self.mainloop.queue_event(self, _GotGraph(artifact)) @@ -348,7 +338,6 @@ class BuildController(distbuild.StateMachine): '(helper id: %s)' % self._helper_id) def _maybe_handle_cache_response(self, event_source, event): - def set_status(artifact): is_in_cache = cache_state[artifact.basename()] artifact.state = BUILT if is_in_cache else UNBUILT @@ -370,15 +359,14 @@ class BuildController(distbuild.StateMachine): map_build_graph(self._artifact, set_status) self.mainloop.queue_event(self, _Annotated()) - count = sum(map_build_graph(self._artifact, - lambda a: 1 if a.state == UNBUILT else 0)) - + unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT]) + total = len([a for _ in self._artifact.walk()]) progress = BuildProgress( self._request['id'], - 'Need to build %d artifacts' % count) + 'Need to build %d artifacts, of %d total' % (unbuilt, total)) self.mainloop.queue_event(BuildController, progress) - if count == 0: + if total == 0: logging.info('There seems to be nothing to build') self.mainloop.queue_event(self, _Built()) diff --git a/distbuild/initiator.py b/distbuild/initiator.py index 378c2cae..548ba496 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -64,7 +64,6 @@ class Initiator(distbuild.StateMachine): self._ref = ref self._morphology = morphology self._original_ref = original_ref - self._steps = None self._step_outputs = {} self.debug_transitions = False @@ -116,7 +115,6 @@ class Initiator(distbuild.StateMachine): 'build-finished': self._handle_build_finished_message, 'build-failed': self._handle_build_failed_message, 'build-progress': self._handle_build_progress_message, - 'build-steps': self._handle_build_steps_message, 'step-started': self._handle_step_started_message, 'step-already-started': self._handle_step_already_started_message, 'step-output': self._handle_step_output_message, @@ -136,12 +134,6 @@ class Initiator(distbuild.StateMachine): def _handle_build_progress_message(self, msg): self._app.status(msg='Progress: %(msgtext)s', msgtext=msg['message']) - def _handle_build_steps_message(self, msg): - self._steps = msg['steps'] - self._app.status( - msg='Build steps in total: %(steps)d', - steps=len(self._steps)) - def _open_output(self, msg): assert msg['step_name'] not in self._step_outputs if self._step_output_dir: diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index 0db13a15..4cd13db3 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -77,8 +77,6 @@ class InitiatorConnection(distbuild.StateMachine): 'idle', self._send_build_failed_message), ('idle', distbuild.BuildController, distbuild.BuildProgress, 'idle', self._send_build_progress_message), - ('idle', distbuild.BuildController, distbuild.BuildSteps, - 'idle', self._send_build_steps_message), ('idle', distbuild.BuildController, distbuild.BuildStepStarted, 'idle', self._send_build_step_started_message), ('idle', distbuild.BuildController, @@ -164,26 +162,6 @@ class InitiatorConnection(distbuild.StateMachine): self.jm.send(msg) self._log_send(msg) - def _send_build_steps_message(self, event_source, event): - - def make_step_dict(artifact): - return { - 'name': distbuild.build_step_name(artifact), - 'build-depends': [ - distbuild.build_step_name(x) - for x in artifact.source.dependencies - ] - } - - if event.id in self.our_ids: - step_names = distbuild.map_build_graph( - event.artifact, make_step_dict) - msg = distbuild.message('build-steps', - id=self._route_map.get_incoming_id(event.id), - steps=step_names) - self.jm.send(msg) - self._log_send(msg) - def _send_build_step_started_message(self, event_source, event): logging.debug('InitiatorConnection: build_step_started: ' 'id=%s step_name=%s worker_name=%s' % diff --git a/distbuild/protocol.py b/distbuild/protocol.py index 8af84960..dee45d17 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -30,10 +30,6 @@ _required_fields = { 'id', 'message', ], - 'build-steps': [ - 'id', - 'steps', - ], 'step-started': [ 'id', 'step_name', -- cgit v1.2.1