summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2015-02-18 11:23:56 +0000
committerSam Thursfield <sam.thursfield@codethink.co.uk>2015-02-18 16:32:50 +0000
commit46a8e0de14ce2cca4c93ec53648d7e01333bbf65 (patch)
treea0bf88f196cbff5f181d98b801f3c41f98df4338
parent51afd2289db20bcf5063ad9512ab1164154ff093 (diff)
downloadmorph-46a8e0de14ce2cca4c93ec53648d7e01333bbf65.tar.gz
distbuild: Remove the build-steps message
This message was hundreds of kilobytes in size, as it contained a recursive list of dependencies for each artifact in the build graph. It was used in the initiator only to print this message: Build steps in total: 592 This message is now gone. The 'Need to build %d artifacts' build-progress message now indicates the total build steps instead: Need to build 300 artifacts, of 592 total This is a compatible change to the distbuild protocol: old initiators will continue to work as normal with new controllers that don't send the build-steps message.
-rw-r--r--distbuild/__init__.py5
-rw-r--r--distbuild/build_controller.py20
-rw-r--r--distbuild/initiator.py8
-rw-r--r--distbuild/initiator_connection.py22
-rw-r--r--distbuild/protocol.py4
5 files changed, 6 insertions, 53 deletions
diff --git a/distbuild/__init__.py b/distbuild/__init__.py
index 0bfae6e9..62a904d4 100644
--- a/distbuild/__init__.py
+++ b/distbuild/__init__.py
@@ -51,9 +51,8 @@ from worker_build_scheduler import (WorkerBuildQueuer,
WorkerBuildFailed,
WorkerBuildStepStarted)
from build_controller import (BuildController, BuildFailed, BuildProgress,
- BuildSteps, BuildStepStarted,
- BuildStepAlreadyStarted, BuildOutput,
- BuildStepFinished, BuildStepFailed,
+ BuildStepStarted, BuildStepAlreadyStarted,
+ BuildOutput, BuildStepFinished, BuildStepFailed,
BuildFinished, BuildCancel,
build_step_name, map_build_graph)
from initiator import Initiator
diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py
index 32da92f4..fd4b2105 100644
--- a/distbuild/build_controller.py
+++ b/distbuild/build_controller.py
@@ -71,13 +71,6 @@ class BuildProgress(object):
self.message_text = message_text
-class BuildSteps(object):
-
- def __init__(self, request_id, artifact):
- self.id = request_id
- self.artifact = artifact
-
-
class BuildStepStarted(object):
def __init__(self, request_id, step_name, worker_name):
@@ -295,9 +288,6 @@ class BuildController(distbuild.StateMachine):
progress = BuildProgress(
self._request['id'], 'Finished computing build graph')
self.mainloop.queue_event(BuildController, progress)
-
- build_steps = BuildSteps(self._request['id'], artifact)
- self.mainloop.queue_event(BuildController, build_steps)
self.mainloop.queue_event(self, _GotGraph(artifact))
@@ -348,7 +338,6 @@ class BuildController(distbuild.StateMachine):
'(helper id: %s)' % self._helper_id)
def _maybe_handle_cache_response(self, event_source, event):
-
def set_status(artifact):
is_in_cache = cache_state[artifact.basename()]
artifact.state = BUILT if is_in_cache else UNBUILT
@@ -370,15 +359,14 @@ class BuildController(distbuild.StateMachine):
map_build_graph(self._artifact, set_status)
self.mainloop.queue_event(self, _Annotated())
- count = sum(map_build_graph(self._artifact,
- lambda a: 1 if a.state == UNBUILT else 0))
-
+ unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT])
+ total = len([a for _ in self._artifact.walk()])
progress = BuildProgress(
self._request['id'],
- 'Need to build %d artifacts' % count)
+ 'Need to build %d artifacts, of %d total' % (unbuilt, total))
self.mainloop.queue_event(BuildController, progress)
- if count == 0:
+ if total == 0:
logging.info('There seems to be nothing to build')
self.mainloop.queue_event(self, _Built())
diff --git a/distbuild/initiator.py b/distbuild/initiator.py
index 378c2cae..548ba496 100644
--- a/distbuild/initiator.py
+++ b/distbuild/initiator.py
@@ -64,7 +64,6 @@ class Initiator(distbuild.StateMachine):
self._ref = ref
self._morphology = morphology
self._original_ref = original_ref
- self._steps = None
self._step_outputs = {}
self.debug_transitions = False
@@ -116,7 +115,6 @@ class Initiator(distbuild.StateMachine):
'build-finished': self._handle_build_finished_message,
'build-failed': self._handle_build_failed_message,
'build-progress': self._handle_build_progress_message,
- 'build-steps': self._handle_build_steps_message,
'step-started': self._handle_step_started_message,
'step-already-started': self._handle_step_already_started_message,
'step-output': self._handle_step_output_message,
@@ -136,12 +134,6 @@ class Initiator(distbuild.StateMachine):
def _handle_build_progress_message(self, msg):
self._app.status(msg='Progress: %(msgtext)s', msgtext=msg['message'])
- def _handle_build_steps_message(self, msg):
- self._steps = msg['steps']
- self._app.status(
- msg='Build steps in total: %(steps)d',
- steps=len(self._steps))
-
def _open_output(self, msg):
assert msg['step_name'] not in self._step_outputs
if self._step_output_dir:
diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py
index 0db13a15..4cd13db3 100644
--- a/distbuild/initiator_connection.py
+++ b/distbuild/initiator_connection.py
@@ -77,8 +77,6 @@ class InitiatorConnection(distbuild.StateMachine):
'idle', self._send_build_failed_message),
('idle', distbuild.BuildController, distbuild.BuildProgress,
'idle', self._send_build_progress_message),
- ('idle', distbuild.BuildController, distbuild.BuildSteps,
- 'idle', self._send_build_steps_message),
('idle', distbuild.BuildController, distbuild.BuildStepStarted,
'idle', self._send_build_step_started_message),
('idle', distbuild.BuildController,
@@ -164,26 +162,6 @@ class InitiatorConnection(distbuild.StateMachine):
self.jm.send(msg)
self._log_send(msg)
- def _send_build_steps_message(self, event_source, event):
-
- def make_step_dict(artifact):
- return {
- 'name': distbuild.build_step_name(artifact),
- 'build-depends': [
- distbuild.build_step_name(x)
- for x in artifact.source.dependencies
- ]
- }
-
- if event.id in self.our_ids:
- step_names = distbuild.map_build_graph(
- event.artifact, make_step_dict)
- msg = distbuild.message('build-steps',
- id=self._route_map.get_incoming_id(event.id),
- steps=step_names)
- self.jm.send(msg)
- self._log_send(msg)
-
def _send_build_step_started_message(self, event_source, event):
logging.debug('InitiatorConnection: build_step_started: '
'id=%s step_name=%s worker_name=%s' %
diff --git a/distbuild/protocol.py b/distbuild/protocol.py
index 8af84960..dee45d17 100644
--- a/distbuild/protocol.py
+++ b/distbuild/protocol.py
@@ -30,10 +30,6 @@ _required_fields = {
'id',
'message',
],
- 'build-steps': [
- 'id',
- 'steps',
- ],
'step-started': [
'id',
'step_name',