Update sentinel support to allow multiple sentinel hosts

Multiple sentinel hosts are now allowed by listing the additional
hosts as multiple 'sentinel_fallback' parameters. These are combined
with the first sentinel host to create a list that is provided to
Sentinel class constructor.

This provides safety in the case when the first (or any other) sentinel
host goes down.

Tests are run twice: once with fallbacks, once without.

Change-Id: Ie0a134f02617e8552a87c8bcd49996daaba07f4e

3 files changed, 49 insertions, 11 deletions

diff --git a/doc/source/drivers.rst b/doc/source/drivers.rst
index b12fa25..75a2524 100644
--- a/doc/source/drivers.rst
+++ b/doc/source/drivers.rst
@@ -15,7 +15,7 @@ API, some of them have different properties:
   on timeout (heartbeats, locks, etc) so are less resilient than other
   backends.
 
-* `redis`_ is a basic implementation and provides some resiliency
+* `redis`_ is a basic implementation and provides reasonable resiliency
   when used with redis-sentinel. A lot of the features provided in tooz are
   based on timeout (heartbeats, locks, etc) so are less resilient than other
   backends.
diff --git a/setup-sentinel-env.sh b/setup-sentinel-env.sh
index 138e0e2..6c935a3 100755
--- a/setup-sentinel-env.sh
+++ b/setup-sentinel-env.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 set -x -e
 
-CONFFILE=$(mktemp -t tooz-sentinel-XXXXXX)
+SENTINEL_PORTS="26381 26382 26383"
+CONFFILES=()
 
 function clean_exit(){
     local error_code="$?"
@@ -10,29 +11,45 @@ function clean_exit(){
         kill $(jobs -p)
     fi
     wait $spawned
-    rm $CONFFILE || true
-    rm /tmp/sentinel.26381.log || true
+    rm /tmp/sentinel.2638[123].log || true
+    rm ${CONFFILES[@]} || true
     return $error_code
 }
 
-cat > $CONFFILE <<EOF
-port 26381
+function write_conf_file() {
+    local port=$1
+    local conffile=$(mktemp -t tooz-sentinel-$port-XXXXXX)
+    cat > $conffile <<EOF
+port $port
 dir /tmp
-sentinel monitor mainbarn 127.0.0.1 6381 1
+sentinel monitor mainbarn 127.0.0.1 6381 2
 sentinel down-after-milliseconds mainbarn 80000
 sentinel parallel-syncs mainbarn 1
 sentinel failover-timeout mainbarn 180000
-logfile /tmp/sentinel.26381.log
+logfile /tmp/sentinel.$port.log
 EOF
+    echo $conffile
+}
 
 trap "clean_exit" EXIT
 
 # If we can't find either redis-server or redis-sentinel, exit.
 redis_bin=$(which redis-server)
 sentinel_bin=$(which redis-sentinel)
+
+# start redis
 $redis_bin --port 6381 &
-$sentinel_bin $CONFFILE &
 
+# start the sentinels
+for port in $SENTINEL_PORTS; do
+    conffile=$(write_conf_file $port)
+    $sentinel_bin $conffile &
+    CONFFILES+=($conffile)
+done
+
+# Test a first time without sentinel fallbacks
 export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
-# Yield execution to venv command
+$*
+# Test a second time with sentinel fallbacks
+export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&sentinel_fallback=localhost:26382&sentinel_fallback=localhost:26383&timeout=5"
 $*
diff --git a/tooz/drivers/redis.py b/tooz/drivers/redis.py
index c11a32f..e6a8df0 100644
--- a/tooz/drivers/redis.py
+++ b/tooz/drivers/redis.py
@@ -151,6 +151,14 @@ class RedisDriver(coordination.CoordinationDriver):
 
       redis://<sentinel host>:<sentinel port>?sentinel=<master name>
 
+    Additional sentinel hosts are listed with mutiple ``sentinel_fallback``
+    parameters as follows:
+
+      redis://<sentinel host>:<sentinel port>?sentinel=<master name>&
+        sentinel_fallback=<other sentinel host>:<sentinel port>&
+        sentinel_fallback=<other sentinel host>:<sentinel port>&
+        sentinel_fallback=<other sentinel host>:<sentinel port>
+
     Further resources/links:
 
     - http://redis.io/
@@ -202,6 +210,10 @@ class RedisDriver(coordination.CoordinationDriver):
         'ssl_certfile',
         'ssl_keyfile',
         'sentinel',
+        'sentinel_fallback',
+    ])
+    _CLIENT_LIST_ARGS = frozenset([
+        'sentinel_fallback',
     ])
     _CLIENT_BOOL_ARGS = frozenset([
         'retry_on_timeout',
@@ -317,6 +329,8 @@ class RedisDriver(coordination.CoordinationDriver):
             # redis://localhost:6379?timeout=5&timeout=2
             if a in cls._CLIENT_BOOL_ARGS:
                 v = strutils.bool_from_string(options[a][-1])
+            elif a in cls._CLIENT_LIST_ARGS:
+                v = options[a]
             elif a in cls._CLIENT_INT_ARGS:
                 v = int(options[a][-1])
             else:
@@ -328,14 +342,21 @@ class RedisDriver(coordination.CoordinationDriver):
         # Ask the sentinel for the current master if there is a
         # sentinel arg.
         if 'sentinel' in kwargs:
+            sentinel_hosts = [
+                tuple(fallback.split(':'))
+                for fallback in kwargs.get('sentinel_fallback', [])
+            ]
+            sentinel_hosts.insert(0, (kwargs['host'], kwargs['port']))
             sentinel_server = sentinel.Sentinel(
-                [(kwargs['host'], kwargs['port'])],
+                sentinel_hosts,
                 socket_timeout=kwargs['socket_timeout'])
             master_host, master_port = sentinel_server.discover_master(
                 kwargs['sentinel'])
             kwargs['host'] = master_host
             kwargs['port'] = master_port
             del kwargs['sentinel']
+            if 'sentinel_fallback' in kwargs:
+                del kwargs['sentinel_fallback']
         return redis.StrictRedis(**kwargs)
 
     def _start(self):