summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Ipsum <richard.ipsum@codethink.co.uk>2014-06-11 16:48:01 +0100
committerRichard Ipsum <richard.ipsum@codethink.co.uk>2014-06-11 16:48:01 +0100
commit2ca28887442c733f2f7a39aa7a844b51643ecb29 (patch)
treea883ff57d8ab08656133e7a77c4760e6c15162cc
parent13edeabd929b094ab2ec7288840e62d31d9feb67 (diff)
parent58e738ceb4b35031793a35e8affc929c7d82f76b (diff)
downloadmorph-2ca28887442c733f2f7a39aa7a844b51643ecb29.tar.gz
Merge branch 'baserock/richardipsum/distbuild_err_when_no_controller2'
Reviewed by: Lars Wirzenius Richard Maw
-rw-r--r--distbuild/__init__.py3
-rw-r--r--distbuild/connection_machine.py58
-rw-r--r--morphlib/buildcommand.py13
3 files changed, 62 insertions, 12 deletions
diff --git a/distbuild/__init__.py b/distbuild/__init__.py
index 7274f6a9..fa0c3831 100644
--- a/distbuild/__init__.py
+++ b/distbuild/__init__.py
@@ -37,7 +37,8 @@ from json_router import JsonRouter
from helper_router import (HelperRouter, HelperRequest, HelperOutput,
HelperResult)
from initiator_connection import (InitiatorConnection, InitiatorDisconnect)
-from connection_machine import ConnectionMachine, Reconnect, StopConnecting
+from connection_machine import (ConnectionMachine, InitiatorConnectionMachine,
+ Reconnect, StopConnecting)
from worker_build_scheduler import (WorkerBuildQueuer,
WorkerConnection,
WorkerBuildRequest,
diff --git a/distbuild/connection_machine.py b/distbuild/connection_machine.py
index 6dec403f..24eec8e9 100644
--- a/distbuild/connection_machine.py
+++ b/distbuild/connection_machine.py
@@ -30,12 +30,13 @@ class Reconnect(object):
class StopConnecting(object):
- pass
-
+ def __init__(self, exception=None):
+ self.exception = exception
class ConnectError(object):
- pass
+ def __init__(self, exception):
+ self.exception = exception
class ProxyEventSource(object):
@@ -63,26 +64,29 @@ class ProxyEventSource(object):
class ConnectionMachine(distbuild.StateMachine):
- def __init__(self, addr, port, machine, extra_args):
- distbuild.StateMachine.__init__(self, 'connecting')
+ def __init__(self, addr, port, machine, extra_args,
+ reconnect_interval=1, max_retries=float('inf')):
+ super(ConnectionMachine, self).__init__('connecting')
self._addr = addr
self._port = port
self._machine = machine
self._extra_args = extra_args
self._socket = None
- self.reconnect_interval = 1
+ self._reconnect_interval = reconnect_interval
+ self._numof_retries = 0
+ self._max_retries = max_retries
def setup(self):
self._sock_proxy = ProxyEventSource()
self.mainloop.add_event_source(self._sock_proxy)
self._start_connect()
- self._timer = distbuild.TimerEventSource(self.reconnect_interval)
+ self._timer = distbuild.TimerEventSource(self._reconnect_interval)
self.mainloop.add_event_source(self._timer)
spec = [
# state, source, event_class, new_state, callback
- ('connecting', self._sock_proxy, distbuild.SocketWriteable,
+ ('connecting', self._sock_proxy, distbuild.SocketWriteable,
'connected', self._connect),
('connecting', self, StopConnecting, None, self._stop),
('connected', self, Reconnect, 'connecting', self._reconnect),
@@ -119,7 +123,11 @@ class ConnectionMachine(distbuild.StateMachine):
'Failed to connect to %s:%s: %s' %
(self._addr, self._port, str(e)))
- self.mainloop.queue_event(self, ConnectError())
+ if self._numof_retries < self._max_retries:
+ self.mainloop.queue_event(self, ConnectError(e))
+ else:
+ self.mainloop.queue_event(self, StopConnecting(e))
+
return
self._sock_proxy.event_source = None
logging.info('Connected to %s:%s' % (self._addr, self._port))
@@ -129,6 +137,8 @@ class ConnectionMachine(distbuild.StateMachine):
def _reconnect(self, event_source, event):
logging.info('Reconnecting to %s:%s' % (self._addr, self._port))
+ self._numof_retries += 1
+
if self._socket is not None:
self._socket.close()
self._timer.stop()
@@ -147,3 +157,33 @@ class ConnectionMachine(distbuild.StateMachine):
self._sock_proxy.event_source.close()
self._sock_proxy.event_source = None
+
+class InitiatorConnectionMachine(ConnectionMachine):
+
+ def __init__(self, app, addr, port, machine, extra_args,
+ reconnect_interval, max_retries):
+
+ self.cm = super(InitiatorConnectionMachine, self)
+ self.cm.__init__(addr, port, machine, extra_args,
+ reconnect_interval, max_retries)
+
+ self.app = app
+
+ def _connect(self, event_source, event):
+ self.app.status(msg='Connecting to %s:%s' % (self._addr, self._port))
+ self.cm._connect(event_source, event)
+
+ def _stop(self, event_source, event):
+ if event.exception:
+ self.app.status(msg="Couldn't connect to %s:%s: %s" %
+ (self._addr, self._port, event.exception.strerror))
+
+ self.cm._stop(event_source, event)
+
+ def _start_timer(self, event_source, event):
+ self.app.status(msg="Couldn't connect to %s:%s: %s" %
+ (self._addr, self._port, event.exception.strerror))
+ self.app.status(msg="Retrying in %d seconds" %
+ self._reconnect_interval)
+
+ self.cm._start_timer(event_source, event)
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 9fab5c07..f68046e3 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -502,6 +502,9 @@ class BuildCommand(object):
class InitiatorBuildCommand(BuildCommand):
+ RECONNECT_INTERVAL = 30 # seconds
+ MAX_RETRIES = 1
+
def __init__(self, app, addr, port):
self.app = app
self.addr = addr
@@ -524,7 +527,13 @@ class InitiatorBuildCommand(BuildCommand):
self.app.status(msg='Starting distributed build')
loop = distbuild.MainLoop()
- cm = distbuild.ConnectionMachine(
- self.addr, self.port, distbuild.Initiator, [self.app] + args)
+ cm = distbuild.InitiatorConnectionMachine(self.app,
+ self.addr,
+ self.port,
+ distbuild.Initiator,
+ [self.app] + args,
+ self.RECONNECT_INTERVAL,
+ self.MAX_RETRIES)
+
loop.add_state_machine(cm)
loop.run()