diff options
Diffstat (limited to 'distbuild/build_controller.py')
| -rw-r--r-- | distbuild/build_controller.py | 135 |
1 files changed, 53 insertions, 82 deletions
diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index 387b410f..3a0dd9b0 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -1,6 +1,6 @@ # distbuild/build_controller.py -- control the steps for one build # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. +# with this program. If not, see <http://www.gnu.org/licenses/>. import logging @@ -37,11 +36,6 @@ class _Start(object): pass class _Annotated(object): pass class _Built(object): pass -class _AnnotationFailed(object): - - def __init__(self, http_status_code, error_msg): - self.http_status_code = http_status_code - self.error_msg = error_msg class _GotGraph(object): @@ -49,11 +43,6 @@ class _GotGraph(object): self.artifact = artifact -class _GraphFailed(object): - - pass - - class BuildCancel(object): def __init__(self, id): @@ -81,13 +70,6 @@ class BuildProgress(object): self.message_text = message_text -class BuildSteps(object): - - def __init__(self, request_id, artifact): - self.id = request_id - self.artifact = artifact - - class BuildStepStarted(object): def __init__(self, request_id, step_name, worker_name): @@ -131,7 +113,7 @@ class _Abort(object): def build_step_name(artifact): '''Return user-comprehensible name for a given artifact.''' - return artifact.name + return artifact.source_name def map_build_graph(artifact, callback): @@ -142,7 +124,7 @@ def map_build_graph(artifact, callback): a = queue.pop() if a not in done: result.append(callback(a)) - queue.extend(a.source.dependencies) + queue.extend(a.dependencies) done.add(a) return result @@ -192,14 +174,13 @@ class BuildController(distbuild.StateMachine): 'graphing', self._maybe_finish_graph), ('graphing', self, _GotGraph, 'annotating', self._start_annotating), - ('graphing', self, _GraphFailed, None, None), + ('graphing', self, BuildFailed, None, None), ('graphing', self._initiator_connection, distbuild.InitiatorDisconnect, None, None), ('annotating', distbuild.HelperRouter, distbuild.HelperResult, 'annotating', self._maybe_handle_cache_response), - ('annotating', self, _AnnotationFailed, None, - self._notify_annotation_failed), + ('annotating', self, BuildFailed, None, None), ('annotating', self, _Annotated, 'building', self._queue_worker_builds), ('annotating', self._initiator_connection, @@ -244,6 +225,29 @@ class BuildController(distbuild.StateMachine): self.mainloop.queue_event(self, _Start()) + def fail(self, reason): + logging.error(reason) + message = BuildFailed(self._request['id'], reason) + + # The message is sent twice so that it can be matched both by state + # transitions listening for this specific controller instance, and by + # state transitions listening for messages from the BuildController + # class that then filter the message based on the request ID field. + self.mainloop.queue_event(self, message) + self.mainloop.queue_event(BuildController, message) + + def _request_command_execution(self, argv, request_id): + '''Tell the controller's distbuild-helper to run a command.''' + if self.mainloop.n_state_machines_of_type(distbuild.HelperRouter) == 0: + self.fail('No distbuild-helper process running on controller!') + + msg = distbuild.message('exec-request', + id=request_id, + argv=argv, + stdin_contents='') + req = distbuild.HelperRequest(msg) + self.mainloop.queue_event(distbuild.HelperRouter, req) + def _start_graphing(self, event_source, event): distbuild.crash_point() @@ -260,14 +264,10 @@ class BuildController(distbuild.StateMachine): ] if 'original_ref' in self._request: argv.append(self._request['original_ref']) - msg = distbuild.message('exec-request', - id=self._idgen.next(), - argv=argv, - stdin_contents='') - self._helper_id = msg['id'] - req = distbuild.HelperRequest(msg) - self.mainloop.queue_event(distbuild.HelperRouter, req) - + + self._helper_id = self._idgen.next() + self._request_command_execution(argv, self._helper_id) + progress = BuildProgress(self._request['id'], 'Computing build graph') self.mainloop.queue_event(BuildController, progress) @@ -281,25 +281,12 @@ class BuildController(distbuild.StateMachine): def _maybe_finish_graph(self, event_source, event): distbuild.crash_point() - def notify_failure(msg_text): - logging.error('Graph creation failed: %s' % msg_text) - - failed = BuildFailed( - self._request['id'], - 'Failed to compute build graph: %s' % msg_text) - self.mainloop.queue_event(BuildController, failed) - - self.mainloop.queue_event(self, _GraphFailed()) - def notify_success(artifact): logging.debug('Graph is finished') progress = BuildProgress( self._request['id'], 'Finished computing build graph') self.mainloop.queue_event(BuildController, progress) - - build_steps = BuildSteps(self._request['id'], artifact) - self.mainloop.queue_event(BuildController, build_steps) self.mainloop.queue_event(self, _GotGraph(artifact)) @@ -308,8 +295,7 @@ class BuildController(distbuild.StateMachine): error_text = self._artifact_error.peek() if event.msg['exit'] != 0 or error_text: - notify_failure('Problem with serialise-artifact: %s' - % error_text) + self.fail(error_text) if event.msg['exit'] != 0: return @@ -317,9 +303,9 @@ class BuildController(distbuild.StateMachine): text = self._artifact_data.peek() try: artifact = distbuild.deserialise_artifact(text) - except ValueError, e: + except ValueError as e: logging.error(traceback.format_exc()) - notify_failure(str(e)) + self.fail('Failed to compute build graph: %s' % e) return notify_success(artifact) @@ -351,7 +337,6 @@ class BuildController(distbuild.StateMachine): '(helper id: %s)' % self._helper_id) def _maybe_handle_cache_response(self, event_source, event): - def set_status(artifact): is_in_cache = cache_state[artifact.basename()] artifact.state = BUILT if is_in_cache else UNBUILT @@ -362,28 +347,25 @@ class BuildController(distbuild.StateMachine): logging.debug('Got cache response: %s' % repr(event.msg)) http_status_code = event.msg['status'] - error_msg = event.msg['body'] if http_status_code != httplib.OK: - logging.debug('Cache request failed with status: %s' - % event.msg['status']) - self.mainloop.queue_event(self, - _AnnotationFailed(http_status_code, error_msg)) + self.fail('Failed to annotate build graph: HTTP request to %s got ' + '%d: %s' % (self._artifact_cache_server, + http_status_code, event.msg['body'])) return cache_state = json.loads(event.msg['body']) map_build_graph(self._artifact, set_status) self.mainloop.queue_event(self, _Annotated()) - count = sum(map_build_graph(self._artifact, - lambda a: 1 if a.state == UNBUILT else 0)) - + unbuilt = len([a for a in self._artifact.walk() if a.state == UNBUILT]) + total = len([a for _ in self._artifact.walk()]) progress = BuildProgress( self._request['id'], - 'Need to build %d artifacts' % count) + 'Need to build %d artifacts, of %d total' % (unbuilt, total)) self.mainloop.queue_event(BuildController, progress) - if count == 0: + if total == 0: logging.info('There seems to be nothing to build') self.mainloop.queue_event(self, _Built()) @@ -391,7 +373,7 @@ class BuildController(distbuild.StateMachine): def is_ready_to_build(artifact): return (artifact.state == UNBUILT and all(a.state == BUILT - for a in artifact.source.dependencies)) + for a in artifact.dependencies)) return [a for a in map_build_graph(self._artifact, lambda a: a) @@ -427,19 +409,19 @@ class BuildController(distbuild.StateMachine): logging.debug( 'Requesting worker-build of %s (%s)' % - (artifact.name, artifact.source.cache_key)) + (artifact.name, artifact.cache_key)) request = distbuild.WorkerBuildRequest(artifact, self._request['id']) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, request) artifact.state = BUILDING - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Chunk artifacts are not built independently # so when we're building any chunk artifact # we're also building all the chunk artifacts # in this source for a in ready: - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILDING @@ -543,7 +525,7 @@ class BuildController(distbuild.StateMachine): def _find_artifact(self, cache_key): artifacts = map_build_graph(self._artifact, lambda a: a) - wanted = [a for a in artifacts if a.source.cache_key == cache_key] + wanted = [a for a in artifacts if a.cache_key == cache_key] if wanted: return wanted[0] else: @@ -569,10 +551,10 @@ class BuildController(distbuild.StateMachine): artifact.state = BUILT def set_state(a): - if a.source == artifact.source: + if a.cache_key == artifact.cache_key: a.state = BUILT - if artifact.source.morphology['kind'] == 'chunk': + if artifact.kind == 'chunk': # Building a single chunk artifact # yields all chunk artifacts for the given source # so we set the state of this source's artifacts @@ -581,14 +563,6 @@ class BuildController(distbuild.StateMachine): self._queue_worker_builds(None, event) - def _notify_annotation_failed(self, event_source, event): - errmsg = ('Failed to annotate build graph: http request got %d: %s' - % (event.http_status_code, event.error_msg)) - - logging.error(errmsg) - failed = BuildFailed(self._request['id'], errmsg) - self.mainloop.queue_event(BuildController, failed) - def _maybe_notify_build_failed(self, event_source, event): distbuild.crash_point() @@ -613,10 +587,7 @@ class BuildController(distbuild.StateMachine): self._request['id'], build_step_name(artifact)) self.mainloop.queue_event(BuildController, step_failed) - build_failed = BuildFailed( - self._request['id'], - 'Building failed for %s' % artifact.name) - self.mainloop.queue_event(BuildController, build_failed) + self.fail('Building failed for %s' % artifact.name) # Cancel any jobs waiting to be executed, since there is no point # running them if this build has failed, it would just waste @@ -640,8 +611,8 @@ class BuildController(distbuild.StateMachine): baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') filename = ('%s.%s.%s' % - (self._artifact.source.cache_key, - self._artifact.source.morphology['kind'], + (self._artifact.cache_key, + self._artifact.kind, self._artifact.name)) url = '%s?filename=%s' % (baseurl, urllib.quote(filename)) finished = BuildFinished(self._request['id'], [url]) |