From ab92ce0343b838b336313f604ea035a60dfcb960 Mon Sep 17 00:00:00 2001 From: Lauren Perry Date: Mon, 30 Mar 2015 12:32:58 +0100 Subject: distbuild: Add distbuild-list-jobs function Add InitiatorListJobs class and list-jobs message template, add distbuild-list-jobs to morph commandlist, send running job information back to initiator, split out handling of build request and list-jobs messages to separate functions and change generating a random integer to UUID for message identification Change-Id: Id02604f2c1201dbc10f6bbd7f501b8ce1ce0deae --- distbuild/__init__.py | 2 +- distbuild/build_controller.py | 10 ++++-- distbuild/initiator.py | 60 +++++++++++++++++++++++++++++++-- distbuild/initiator_connection.py | 64 +++++++++++++++++++++++++----------- distbuild/protocol.py | 6 ++++ morphlib/plugins/distbuild_plugin.py | 42 +++++++++++++++++++++++ 6 files changed, 158 insertions(+), 26 deletions(-) diff --git a/distbuild/__init__.py b/distbuild/__init__.py index fc74d480..271b5def 100644 --- a/distbuild/__init__.py +++ b/distbuild/__init__.py @@ -54,7 +54,7 @@ from build_controller import (BuildController, BuildFailed, BuildProgress, BuildOutput, BuildStepFinished, BuildStepFailed, BuildFinished, BuildCancel, build_step_name, map_build_graph) -from initiator import Initiator +from initiator import (Initiator, InitiatorListJobs) from protocol import message from crashpoint import (crash_point, add_crash_condition, add_crash_conditions, diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index 6058862c..3971fe68 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -180,6 +180,12 @@ class BuildController(distbuild.StateMachine): return '' % (id(self), self._request['id']) + def get_initiator_connection(self): + return self._initiator_connection + + def get_request(self): + return self._request + def setup(self): distbuild.crash_point() @@ -483,8 +489,8 @@ class BuildController(distbuild.StateMachine): def _maybe_notify_initiator_disconnected(self, event_source, event): if event.id != self._request['id']: - logging.debug('Heard initiator disconnect with event id %d ' - 'but our request id is %d', + logging.debug('Heard initiator disconnect with event id %s ' + 'but our request id is %s', event.id, self._request['id']) return # not for us diff --git a/distbuild/initiator.py b/distbuild/initiator.py index 48299a3d..eee25313 100644 --- a/distbuild/initiator.py +++ b/distbuild/initiator.py @@ -19,7 +19,7 @@ import cliapp import itertools import logging import os -import random +import uuid import time import distbuild @@ -92,7 +92,7 @@ class Initiator(distbuild.StateMachine): ] self.add_transitions(spec) - random_id = random.randint(0, 2**32-1) + msg_uuid = uuid.uuid4().hex self._app.status( msg='Requesting build of %(repo)s %(ref)s %(morph)s', @@ -100,7 +100,7 @@ class Initiator(distbuild.StateMachine): ref=self._ref, morph=self._morphology) msg = distbuild.message('build-request', - id=random_id, + id=msg_uuid, repo=self._repo_name, ref=self._ref, morphology=self._morphology, @@ -247,3 +247,57 @@ class Initiator(distbuild.StateMachine): f.close() self._step_outputs = {} + + +class InitiatorListJobs(distbuild.StateMachine): + + def __init__(self, cm, conn, app): + distbuild.StateMachine.__init__(self, 'waiting') + self._cm = cm + self._conn = conn + self._app = app + + def setup(self): + distbuild.crash_point() + + self._jm = distbuild.JsonMachine(self._conn) + self.mainloop.add_state_machine(self._jm) + logging.debug('initiator: _jm=%s' % repr(self._jm)) + + spec = [ + # state, source, event_class, new_state, callback + ('waiting', self._jm, distbuild.JsonEof, None, self._terminate), + ('waiting', self._jm, distbuild.JsonNewMessage, None, + self._handle_json_message), + ] + self.add_transitions(spec) + + msg_uuid = uuid.uuid4().hex + + self._app.status(msg='Requesting currently running distbuilds.') + msg = distbuild.message('list-requests', + id=msg_uuid, + ) + self._jm.send(msg) + logging.debug('Initiator: sent to controller: %s', repr(msg)) + + def _handle_json_message(self, event_source, event): + distbuild.crash_point() + + logging.debug('Initiator: from controller: %s', str(event.msg)) + + handlers = { + 'list-request-output': self._handle_list_request_output, + } + + handler = handlers[event.msg['type']] + handler(event.msg) + + def _handle_list_request_output(self, msg): + self._app.status(msg=str(msg['message'])) + self.mainloop.queue_event(self._cm, distbuild.StopConnecting()) + self._jm.close() + + def _terminate(self, event_source, event): + self.mainloop.queue_event(self._cm, distbuild.StopConnecting()) + self._jm.close() diff --git a/distbuild/initiator_connection.py b/distbuild/initiator_connection.py index 8d6c1f4c..54322f6a 100644 --- a/distbuild/initiator_connection.py +++ b/distbuild/initiator_connection.py @@ -99,30 +99,54 @@ class InitiatorConnection(distbuild.StateMachine): try: if event.msg['type'] == 'build-request': - if (event.msg.get('protocol_version') != - distbuild.protocol.VERSION): - msg = distbuild.message('build-failed', - id=event.msg['id'], - reason=('Protocol version mismatch between server & ' - 'initiator: distbuild network uses distbuild ' - 'protocol version %i, but client uses version' - ' %i.' % (distbuild.protocol.VERSION, - event.msg.get('protocol_version')))) - self.jm.send(msg) - self._log_send(msg) - return - new_id = self._idgen.next() - self.our_ids.add(new_id) - self._route_map.add(event.msg['id'], new_id) - event.msg['id'] = new_id - build_controller = distbuild.BuildController( - self, event.msg, self.artifact_cache_server, - self.morph_instance) - self.mainloop.add_state_machine(build_controller) + self._handle_build_request(event) + elif event.msg['type'] == 'list-requests': + self._handle_list_requests(event) + else: + logging.error('Invalid message type: %s', event.msg) except (KeyError, ValueError) as ex: logging.error('Invalid message from initiator: %s: exception %s', event.msg, ex) + def _handle_build_request(self, event): + if event.msg.get('protocol_version') != distbuild.protocol.VERSION: + msg = distbuild.message('build-failed', + id=event.msg['id'], + reason=('Protocol version mismatch between server & initiator:' + ' distbuild network uses distbuild protocol version %i' + ', but client uses version %i.' % ( + distbuild.protocol.VERSION, + event.msg.get('protocol_version')))) + self.jm.send(msg) + self._log_send(msg) + return + new_id = self._idgen.next() + self.our_ids.add(new_id) + self._route_map.add(event.msg['id'], new_id) + event.msg['id'] = new_id + build_controller = distbuild.BuildController( + self, event.msg, self.artifact_cache_server, + self.morph_instance) + self.mainloop.add_state_machine(build_controller) + + def _handle_list_requests(self, event): + requests = self.mainloop.state_machines_of_type( + distbuild.BuildController) + output_msg = [] + output_msg.append('%s distbuild requests(s) currently in progress' % + len(requests)) + for build in requests: + output_msg.append('Build request ID: %s\n Initiator: %s\n Repo: ' + '%s\n Ref: %s\n Component: %s' + % (build.get_request()['id'], + build.get_initiator_connection().initiator_name, + build.get_request()['repo'], + build.get_request()['ref'], + build.get_request()['morphology'])) + msg = distbuild.message('list-request-output', + message=('\n\n'.join(output_msg))) + self.jm.send(msg) + def _disconnect(self, event_source, event): for id in self.our_ids: logging.debug('InitiatorConnection: %s: InitiatorDisconnect(%s)', diff --git a/distbuild/protocol.py b/distbuild/protocol.py index 268dcbf6..0f936946 100644 --- a/distbuild/protocol.py +++ b/distbuild/protocol.py @@ -85,6 +85,12 @@ _required_fields = { 'headers', 'body', ], + 'list-requests': [ + 'id', + ], + 'list-request-output': [ + 'message', + ], } diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index ac3957f3..8aaead10 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -40,6 +40,48 @@ class DistbuildOptionsPlugin(cliapp.Plugin): pass +class DistbuildListJobsPlugin(cliapp.Plugin): + + RECONNECT_INTERVAL = 30 # seconds + MAX_RETRIES = 1 + + def enable(self): + self.app.add_subcommand('distbuild-list-jobs', + self.distbuild_list_jobs, arg_synopsis='') + + def disable(self): + pass + + def distbuild_list_jobs(self, args): + '''Display a list of currently running distbuilds. + + Lists all distbuilds running on a given address and port, as set in + the client machine's morph.conf file + + Example output: + + '1 distbuild build request(s) currently in progress + Initiator connection (address:port): localhost:7878 + Build request message: {'repo': 'baserock:baserock/definitions', + 'original_ref': 'BRANCH_NAME', 'ref': 'SHA1', 'morphology': + 'systems/devel-system-x86_64-generic.morph', 'protocol_version': 1, + 'type': 'build-request', 'id': 'InitiatorConnection-x'}' + Build request ID: InitiatorConnection-x + + ''' + + addr = self.app.settings['controller-initiator-address'] + port = self.app.settings['controller-initiator-port'] + icm = distbuild.InitiatorConnectionMachine(self.app, addr, port, + distbuild.InitiatorListJobs, + [self.app], + self.RECONNECT_INTERVAL, + self.MAX_RETRIES) + loop = distbuild.MainLoop() + loop.add_state_machine(icm) + loop.run() + + class SerialiseArtifactPlugin(cliapp.Plugin): def enable(self): -- cgit v1.2.1