Add support for an optional redis-sentinel0.10

If sentinel support is desired then the connection URI should take
the form:

   redis://<sentinel host>:<sentinel port>?sentinel=<master name>

This will be parsed to connect to a Sentinel server to discover a
master. The response will be used to make a connection to the
current master.

The setup-sentinel-env.sh script will exit with an error if either
the redis-server or sentinel cannot be found. When the script is
done it will clean up the mess it makes.

Change-Id: I205ed88c72587d9f5f916d344c44192bee2aa41a

4 files changed, 72 insertions, 4 deletions

diff --git a/doc/source/drivers.rst b/doc/source/drivers.rst
index 5b828c6..b12fa25 100644
--- a/doc/source/drivers.rst
+++ b/doc/source/drivers.rst
@@ -15,9 +15,10 @@ API, some of them have different properties:
   on timeout (heartbeats, locks, etc) so are less resilient than other
   backends.
 
-* `redis`_ is a basic implementation and provides little resiliency.
-  A lot of the features provided in tooz are based on timeout (heartbeats,
-  locks, etc) so are less resilient than other backends.
+* `redis`_ is a basic implementation and provides some resiliency
+  when used with redis-sentinel. A lot of the features provided in tooz are
+  based on timeout (heartbeats, locks, etc) so are less resilient than other
+  backends.
 
 * `ipc` is based on Posix IPC and only implements a lock mechanism for now, and
   some basic group primitives (with huge limitations). The lock can only be
diff --git a/setup-sentinel-env.sh b/setup-sentinel-env.sh
new file mode 100755
index 0000000..138e0e2
--- /dev/null
+++ b/setup-sentinel-env.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+set -x -e
+
+CONFFILE=$(mktemp -t tooz-sentinel-XXXXXX)
+
+function clean_exit(){
+    local error_code="$?"
+    local spawned=$(jobs -p)
+    if [ -n "$spawned" ]; then
+        kill $(jobs -p)
+    fi
+    wait $spawned
+    rm $CONFFILE || true
+    rm /tmp/sentinel.26381.log || true
+    return $error_code
+}
+
+cat > $CONFFILE <<EOF
+port 26381
+dir /tmp
+sentinel monitor mainbarn 127.0.0.1 6381 1
+sentinel down-after-milliseconds mainbarn 80000
+sentinel parallel-syncs mainbarn 1
+sentinel failover-timeout mainbarn 180000
+logfile /tmp/sentinel.26381.log
+EOF
+
+trap "clean_exit" EXIT
+
+# If we can't find either redis-server or redis-sentinel, exit.
+redis_bin=$(which redis-server)
+sentinel_bin=$(which redis-sentinel)
+$redis_bin --port 6381 &
+$sentinel_bin $CONFFILE &
+
+export TOOZ_TEST_REDIS_URL="redis://localhost:26381?sentinel=mainbarn&timeout=5"
+# Yield execution to venv command
+$*
diff --git a/tooz/drivers/redis.py b/tooz/drivers/redis.py
index 333043e..c11a32f 100644
--- a/tooz/drivers/redis.py
+++ b/tooz/drivers/redis.py
@@ -26,6 +26,7 @@ from oslo.utils import strutils
 import redis
 from redis import exceptions
 from redis import lock as redis_locks
+from redis import sentinel
 import six
 from six.moves import map as compat_map
 from six.moves import zip as compat_zip
@@ -143,6 +144,13 @@ class RedisDriver(coordination.CoordinationDriver):
       some notion of HA (values *can* be lost when a failover transition
       occurs).
 
+    To use a sentinel the connection URI must point to the Sentinel server.
+    At connection time the sentinel will be asked for the current IP and port
+    of the master and then connect there. The connection URI for sentinel
+    should be written as follows::
+
+      redis://<sentinel host>:<sentinel port>?sentinel=<master name>
+
     Further resources/links:
 
     - http://redis.io/
@@ -193,6 +201,7 @@ class RedisDriver(coordination.CoordinationDriver):
         'ssl',
         'ssl_certfile',
         'ssl_keyfile',
+        'sentinel',
     ])
     _CLIENT_BOOL_ARGS = frozenset([
         'retry_on_timeout',
@@ -315,6 +324,18 @@ class RedisDriver(coordination.CoordinationDriver):
             kwargs[a] = v
         if 'socket_timeout' not in kwargs:
             kwargs['socket_timeout'] = default_socket_timeout
+
+        # Ask the sentinel for the current master if there is a
+        # sentinel arg.
+        if 'sentinel' in kwargs:
+            sentinel_server = sentinel.Sentinel(
+                [(kwargs['host'], kwargs['port'])],
+                socket_timeout=kwargs['socket_timeout'])
+            master_host, master_port = sentinel_server.discover_master(
+                kwargs['sentinel'])
+            kwargs['host'] = master_host
+            kwargs['port'] = master_port
+            del kwargs['sentinel']
         return redis.StrictRedis(**kwargs)
 
     def _start(self):
diff --git a/tox.ini b/tox.ini
index 35463e7..67f231c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 minversion = 1.6
 skipsdist = True
-envlist = py26,py27,py34,py27-zookeeper,py34-zookeeper,py27-redis,py34-redis,py27-memcached,py34-memcached,py27-postgresql,py34-postgresql,py27-mysql,py34-mysql,pep8
+envlist = py26,py27,py34,py27-zookeeper,py34-zookeeper,py27-redis,py34-redis,py27-sentinel,py34-sentinel,py27-memcached,py34-memcached,py27-postgresql,py34-postgresql,py27-mysql,py34-mysql,pep8
 
 [testenv]
 deps = -r{toxinidir}/requirements.txt
@@ -42,6 +42,14 @@ deps = {[testenv:py34]deps}
 basepython = python3.4
 commands = {toxinidir}/setup-redis-env.sh python setup.py testr --slowest --testr-args="{posargs}"
 
+[testenv:py27-sentinel]
+commands = {toxinidir}/setup-sentinel-env.sh python setup.py testr --slowest --testr-args="{posargs}"
+
+[testenv:py34-sentinel]
+deps = {[testenv:py34]deps}
+basepython = python3.4
+commands = {toxinidir}/setup-sentinel-env.sh python setup.py testr --slowest --testr-args="{posargs}"
+
 [testenv:py27-memcached]
 commands = {toxinidir}/setup-memcached-env.sh python setup.py testr --slowest --testr-args="{posargs}"