From 4941d1dd806d1751b1ac0f7a695ec2b6dce93ba0 Mon Sep 17 00:00:00 2001 From: Lauren Perry Date: Fri, 10 Apr 2015 15:29:06 +0100 Subject: distbuild: Add distbuild status command Adds a command to get the status of all recently ran distbuilds for a given server (e.g. Running, Finished, Failed, Cancelled), so as to tell if a build running via distbuild-start has finished or otherwise exited without going through the server's log files Change-Id: I5ce9fe54ae7b1bd8fe3e0d629f615042be8827ed --- distbuild/__init__.py | 2 +- distbuild/build_controller.py | 12 ++++++++ distbuild/initiator.py | 54 ++++++++++++++++++++++++++++++++++++ distbuild/initiator_connection.py | 34 +++++++++++++++++------ distbuild/mainloop.py | 4 ++- distbuild/protocol.py | 4 +++ morphlib/plugins/distbuild_plugin.py | 47 +++++++++++++++++++++++++++++++ 7 files changed, 147 insertions(+), 10 deletions(-) diff --git a/distbuild/__init__.py b/distbuild/__init__.py index aaf9ae41..62cf414c 100644 --- a/distbuild/__init__.py +++ b/distbuild/__init__.py @@ -56,7 +56,7 @@ from build_controller import (BuildController, BuildFailed, BuildProgress, BuildFinished, BuildCancel, BuildStarted, build_step_name, map_build_graph) from initiator import (Initiator, InitiatorStart, InitiatorCancel, - InitiatorListJobs) + InitiatorListJobs, InitiatorStatus) from protocol import message from crashpoint import (crash_point, add_crash_condition, add_crash_conditions, diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index d879ffea..e38b25b7 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -183,6 +183,11 @@ class BuildController(distbuild.StateMachine): self.debug_transitions = False self.debug_graph_state = False self.allow_detach = build_request_message['allow_detach'] + self.build_info = { + 'id': build_request_message['id'], + 'morphology': build_request_message['morphology'], + 'status': 'Computing build graph' + } def __repr__(self): return '' % (id(self), @@ -535,6 +540,7 @@ class BuildController(distbuild.StateMachine): logging.debug("BuildController %r: initiator id %s cancelled", self, event.id) + self.build_info['status'] = 'Cancelled' cancel_pending = distbuild.WorkerCancelPending(event.id) self.mainloop.queue_event(distbuild.WorkerBuildQueuer, cancel_pending) @@ -553,6 +559,7 @@ class BuildController(distbuild.StateMachine): # This is not the event you are looking for. return + self.build_info['status'] = 'Waiting for a worker to become available' progress = BuildProgress( self._request['id'], 'Ready to build %s: waiting for a worker to become available' @@ -573,6 +580,7 @@ class BuildController(distbuild.StateMachine): return logging.debug('BC: got build step started: %s' % artifact.name) + self.build_info['status'] = 'Building %s' % artifact.name started = BuildStepStarted( self._request['id'], build_step_name(artifact), event.worker_name) self.mainloop.queue_event(BuildController, started) @@ -585,6 +593,7 @@ class BuildController(distbuild.StateMachine): artifact = self._find_artifact(event.artifact_cache_key) logging.debug('BC: got build step already started: %s' % artifact.name) + self.build_info['status'] = 'Building %s' % artifact.name started = BuildStepAlreadyStarted( self._request['id'], build_step_name(artifact), event.worker_name) self.mainloop.queue_event(BuildController, started) @@ -692,6 +701,8 @@ class BuildController(distbuild.StateMachine): self.fail('Building failed for %s' % artifact.name) + self.build_info['status'] = 'Failed building %s' % artifact.name + # Cancel any jobs waiting to be executed, since there is no point # running them if this build has failed, it would just waste # resources @@ -711,6 +722,7 @@ class BuildController(distbuild.StateMachine): distbuild.crash_point() logging.debug('Notifying initiator of successful build') + self.build_info['status'] = 'Finished' baseurl = urlparse.urljoin( self._artifact_cache_server, '/1.0/artifacts') urls = [] diff --git a/distbuild/initiator.py b/distbuild/initiator.py index 40b56a9d..359c9dbf 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -431,3 +431,57 @@ class InitiatorListJobs(distbuild.StateMachine): def _terminate(self, event_source, event): self.mainloop.queue_event(self._cm, distbuild.StopConnecting()) self._jm.close() + + +class InitiatorStatus(distbuild.StateMachine): + + def __init__(self, cm, conn, app, job_id): + distbuild.StateMachine.__init__(self, 'waiting') + self._cm = cm + self._conn = conn + self._app = app + self._job_id = job_id + + def setup(self): + distbuild.crash_point() + + self._jm = distbuild.JsonMachine(self._conn) + self.mainloop.add_state_machine(self._jm) + logging.debug('initiator: _jm=%s' % repr(self._jm)) + + spec = [ + # state, source, event_class, new_state, callback + ('waiting', self._jm, distbuild.JsonEof, None, self._terminate), + ('waiting', self._jm, distbuild.JsonNewMessage, None, + self._handle_json_message), + ] + self.add_transitions(spec) + + self._app.status(msg='Requesting status of recent build requests.') + msg = distbuild.message('build-status', + id=self._job_id, + protocol_version=distbuild.protocol.VERSION, + ) + self._jm.send(msg) + logging.debug('Initiator: sent to controller: %s', repr(msg)) + + def _handle_json_message(self, event_source, event): + distbuild.crash_point() + + logging.debug('Initiator: from controller: %s', str(event.msg)) + + handlers = { + 'request-output': self._handle_request_output, + } + + handler = handlers[event.msg['type']] + handler(event.msg) + + def _handle_request_output(self, msg): + self._app.status(msg=str(msg['message'])) + self.mainloop.queue_event(self._cm, distbuild.StopConnecting()) + self._jm.close() + + def _terminate(self, event_source, event): + self.mainloop.queue_event(self._cm, distbuild.StopConnecting()) + self._jm.close() diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index 718686dc..7fd45d4d 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -107,6 +107,12 @@ class InitiatorConnection(distbuild.StateMachine): logging.debug('InitiatorConnection: from %s: %r', self.initiator_name, event.msg) + msg_handler = { + 'build-request': self._handle_build_request, + 'list-requests': self._handle_list_requests, + 'build-cancel': self._handle_build_cancel, + 'build-status': self._handle_build_status, + } try: if event.msg.get('protocol_version') != distbuild.protocol.VERSION: msg = distbuild.message('build-failed', @@ -123,14 +129,7 @@ class InitiatorConnection(distbuild.StateMachine): self.jm.send(msg) self._log_send(msg) return - if event.msg['type'] == 'build-request': - self._handle_build_request(event) - elif event.msg['type'] == 'list-requests': - self._handle_list_requests(event) - elif event.msg['type'] == 'build-cancel': - self._handle_build_cancel(event) - else: - logging.error('Invalid message type: %s', event.msg) + msg_handler[event.msg['type']](event) except (KeyError, ValueError) as ex: logging.error('Invalid message from initiator: %s: exception %s', event.msg, ex) @@ -144,6 +143,7 @@ class InitiatorConnection(distbuild.StateMachine): self, event.msg, self.artifact_cache_server, self.morph_instance) self.mainloop.add_state_machine(build_controller) + self.mainloop.build_info.append(build_controller.build_info) def _handle_list_requests(self, event): requests = self.mainloop.state_machines_of_type( @@ -181,6 +181,24 @@ class InitiatorConnection(distbuild.StateMachine): 'running build IDs.')) self.jm.send(msg) + def _handle_build_status(self, event): + for build_info in self.mainloop.build_info: + if build_info['id'] == event.msg['id']: + msg = distbuild.message('request-output', + message=('\nBuild request ID: %s\n System build: %s\n ' + 'Build status: %s' % (build_info['id'], + build_info['morphology'], + build_info['status']))) + + self.jm.send(msg) + break + else: + msg = distbuild.message('request-output', message=('Given ' + 'build-request ID does not match any ' + 'recent build IDs (the status information ' + 'for this build may have expired).')) + self.jm.send(msg) + def _disconnect(self, event_source, event): for id in self.our_ids: logging.debug('InitiatorConnection: %s: InitiatorDisconnect(%s)', diff --git a/distbuild/mainloop.py b/distbuild/mainloop.py index d48d60ec..4d5a2b61 100644 --- a/distbuild/mainloop.py +++ b/distbuild/mainloop.py @@ -19,6 +19,7 @@ import fcntl import logging import os import select +import collections class MainLoop(object): @@ -40,6 +41,7 @@ class MainLoop(object): self._machines = [] self._sources = [] self._events = [] + self.build_info = collections.deque(maxlen=1000) self.dump_filename = None def add_state_machine(self, machine): @@ -51,7 +53,7 @@ class MainLoop(object): filename = '%s%s.dot' % (self.dump_filename, machine.__class__.__name__) machine.dump_dot(filename) - + def remove_state_machine(self, machine): logging.debug('MainLoop.remove_state_machine: %s' % machine) self._machines.remove(machine) diff --git a/distbuild/protocol.py b/distbuild/protocol.py index 8f533e75..eab16141 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -103,6 +103,10 @@ _required_fields = { 'id', 'protocol_version', ], + 'build-status': [ + 'id', + 'protocol_version', + ], } diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 68a80784..71d83dfe 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -82,6 +82,53 @@ class DistbuildCancel(cliapp.Plugin): loop.run() +class DistbuildStatusPlugin(cliapp.Plugin): + + RECONNECT_INTERVAL = 30 # seconds + MAX_RETRIES = 1 + + def enable(self): + self.app.add_subcommand('distbuild-status', self.distbuild_status, + arg_synopsis='ID') + + def disable(self): + pass + + def distbuild_status(self, args): + '''Displays build status of recent distbuild requests. + + Lists last known build status for all distbuilds (e.g. Building, + Failed, Finished, Cancelled) on a given distbuild server as set in + /etc/morph.conf + + Example: + + morph distbuild-status InitiatorConnection-1 + + Example output: + + Build request ID: InitiatorConnection-1 + System build: systems/devel-system-x86_64-generic.morph + Build status: Building stage1-binutils-misc + + ''' + + if len(args) != 1: + raise cliapp.AppException( + 'usage: morph distbuild-status ') + + addr = self.app.settings['controller-initiator-address'] + port = self.app.settings['controller-initiator-port'] + icm = distbuild.InitiatorConnectionMachine(self.app, addr, port, + distbuild.InitiatorStatus, + [self.app] + args, + self.RECONNECT_INTERVAL, + self.MAX_RETRIES) + loop = distbuild.MainLoop() + loop.add_state_machine(icm) + loop.run() + + class DistbuildListJobsPlugin(cliapp.Plugin): RECONNECT_INTERVAL = 30 # seconds -- cgit v1.2.1