From 33bef5a71770b25fe22053267551a10c95c71f7b Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 11 Jun 2014 14:48:26 +0100 Subject: Add max_retries option By default there is no limit on the number of reconnection attempts. We make the reconnect_interval a parameter, but the default interval remains 1 second. --- distbuild/connection_machine.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/distbuild/connection_machine.py b/distbuild/connection_machine.py index 6dec403f..cce83a40 100644 --- a/distbuild/connection_machine.py +++ b/distbuild/connection_machine.py @@ -63,26 +63,29 @@ class ProxyEventSource(object): class ConnectionMachine(distbuild.StateMachine): - def __init__(self, addr, port, machine, extra_args): + def __init__(self, addr, port, machine, extra_args, + reconnect_interval=1, max_retries=float('inf')): distbuild.StateMachine.__init__(self, 'connecting') self._addr = addr self._port = port self._machine = machine self._extra_args = extra_args self._socket = None - self.reconnect_interval = 1 + self._reconnect_interval = reconnect_interval + self._numof_retries = 0 + self._max_retries = max_retries def setup(self): self._sock_proxy = ProxyEventSource() self.mainloop.add_event_source(self._sock_proxy) self._start_connect() - self._timer = distbuild.TimerEventSource(self.reconnect_interval) + self._timer = distbuild.TimerEventSource(self._reconnect_interval) self.mainloop.add_event_source(self._timer) spec = [ # state, source, event_class, new_state, callback - ('connecting', self._sock_proxy, distbuild.SocketWriteable, + ('connecting', self._sock_proxy, distbuild.SocketWriteable, 'connected', self._connect), ('connecting', self, StopConnecting, None, self._stop), ('connected', self, Reconnect, 'connecting', self._reconnect), @@ -119,7 +122,11 @@ class ConnectionMachine(distbuild.StateMachine): 'Failed to connect to %s:%s: %s' % (self._addr, self._port, str(e))) - self.mainloop.queue_event(self, ConnectError()) + if self._numof_retries < self._max_retries: + self.mainloop.queue_event(self, ConnectError()) + else: + self.mainloop.queue_event(self, StopConnecting()) + return self._sock_proxy.event_source = None logging.info('Connected to %s:%s' % (self._addr, self._port)) @@ -129,6 +136,8 @@ class ConnectionMachine(distbuild.StateMachine): def _reconnect(self, event_source, event): logging.info('Reconnecting to %s:%s' % (self._addr, self._port)) + self._numof_retries += 1 + if self._socket is not None: self._socket.close() self._timer.stop() -- cgit v1.2.1 From 3fa8e6fe62b897b90b8a6c3ab518ef702eeda47e Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 11 Jun 2014 16:33:40 +0100 Subject: Add InitiatorConnectionMachine The InitiatorConnectionMachine wraps the ConnectionMachine, so we can continue to use ConnectionMachine without providing it with an app. --- distbuild/connection_machine.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/distbuild/connection_machine.py b/distbuild/connection_machine.py index cce83a40..0faa7631 100644 --- a/distbuild/connection_machine.py +++ b/distbuild/connection_machine.py @@ -30,12 +30,13 @@ class Reconnect(object): class StopConnecting(object): - pass - + def __init__(self, exception=None): + self.exception = exception class ConnectError(object): - pass + def __init__(self, exception): + self.exception = exception class ProxyEventSource(object): @@ -123,9 +124,9 @@ class ConnectionMachine(distbuild.StateMachine): (self._addr, self._port, str(e))) if self._numof_retries < self._max_retries: - self.mainloop.queue_event(self, ConnectError()) + self.mainloop.queue_event(self, ConnectError(e)) else: - self.mainloop.queue_event(self, StopConnecting()) + self.mainloop.queue_event(self, StopConnecting(e)) return self._sock_proxy.event_source = None @@ -156,3 +157,33 @@ class ConnectionMachine(distbuild.StateMachine): self._sock_proxy.event_source.close() self._sock_proxy.event_source = None + +class InitiatorConnectionMachine(ConnectionMachine): + + def __init__(self, app, addr, port, machine, extra_args, + reconnect_interval, max_retries): + + self.cm = super(InitiatorConnectionMachine, self) + self.cm.__init__(addr, port, machine, extra_args, + reconnect_interval, max_retries) + + self.app = app + + def _connect(self, event_source, event): + self.app.status(msg='Connecting to %s:%s' % (self._addr, self._port)) + self.cm._connect(event_source, event) + + def _stop(self, event_source, event): + if event.exception: + self.app.status(msg="Couldn't connect to %s:%s: %s" % + (self._addr, self._port, event.exception.strerror)) + + self.cm._stop(event_source, event) + + def _start_timer(self, event_source, event): + self.app.status(msg="Couldn't connect to %s:%s: %s" % + (self._addr, self._port, event.exception.strerror)) + self.app.status(msg="Retrying in %d seconds" % + self._reconnect_interval) + + self.cm._start_timer(event_source, event) -- cgit v1.2.1 From 34dd16174b71832b9d005933cbe790e6b69ce2d3 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 11 Jun 2014 14:49:28 +0100 Subject: Use super This change is made just for consistency. --- distbuild/connection_machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distbuild/connection_machine.py b/distbuild/connection_machine.py index 0faa7631..24eec8e9 100644 --- a/distbuild/connection_machine.py +++ b/distbuild/connection_machine.py @@ -66,7 +66,7 @@ class ConnectionMachine(distbuild.StateMachine): def __init__(self, addr, port, machine, extra_args, reconnect_interval=1, max_retries=float('inf')): - distbuild.StateMachine.__init__(self, 'connecting') + super(ConnectionMachine, self).__init__('connecting') self._addr = addr self._port = port self._machine = machine -- cgit v1.2.1 From 3e7af0afebff07df01db4785c1b82614e0aeaa9a Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Fri, 6 Jun 2014 12:38:23 +0100 Subject: Import InitiatorConnectionMachine --- distbuild/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distbuild/__init__.py b/distbuild/__init__.py index 7274f6a9..fa0c3831 100644 --- a/distbuild/__init__.py +++ b/distbuild/__init__.py @@ -37,7 +37,8 @@ from json_router import JsonRouter from helper_router import (HelperRouter, HelperRequest, HelperOutput, HelperResult) from initiator_connection import (InitiatorConnection, InitiatorDisconnect) -from connection_machine import ConnectionMachine, Reconnect, StopConnecting +from connection_machine import (ConnectionMachine, InitiatorConnectionMachine, + Reconnect, StopConnecting) from worker_build_scheduler import (WorkerBuildQueuer, WorkerConnection, WorkerBuildRequest, -- cgit v1.2.1 From 58e738ceb4b35031793a35e8affc929c7d82f76b Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 5 Jun 2014 15:50:52 +0100 Subject: Make buildcommand use InitiatorConnectionMachine After a failed attempt to connect to the controller node the initiator will wait 30 seconds before attempting a reconnect, if this reconnect fails the initiator gives up. --- morphlib/buildcommand.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 9fab5c07..f68046e3 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -502,6 +502,9 @@ class BuildCommand(object): class InitiatorBuildCommand(BuildCommand): + RECONNECT_INTERVAL = 30 # seconds + MAX_RETRIES = 1 + def __init__(self, app, addr, port): self.app = app self.addr = addr @@ -524,7 +527,13 @@ class InitiatorBuildCommand(BuildCommand): self.app.status(msg='Starting distributed build') loop = distbuild.MainLoop() - cm = distbuild.ConnectionMachine( - self.addr, self.port, distbuild.Initiator, [self.app] + args) + cm = distbuild.InitiatorConnectionMachine(self.app, + self.addr, + self.port, + distbuild.Initiator, + [self.app] + args, + self.RECONNECT_INTERVAL, + self.MAX_RETRIES) + loop.add_state_machine(cm) loop.run() -- cgit v1.2.1