Enable retries in redis driver

this is followup to Iaab5ce609c0dcf7085f5dd43efbd37eb4b88f17b

actually retry for specified number of retries instead of raising
error on first ConnectionError

Conflicts:
  tooz/drivers/redis.py

NOTE(stephenfin): Conflicts are due to the changes we had to make in
another backport, change Iaab5ce609c0dcf7085f5dd43efbd37eb4b88f17b, to
keep the pep8 job happy. Those changed lines are replaced here.

Change-Id: Ibca3f568b65dfea252da4b67f6d5105ba7f1ecb1
(cherry picked from commit 47c4d56e446a49726e7145c6cbc3bb7620a431f7)
(cherry picked from commit 11526e594c7e7cef8b5a4a4220f100d76a418036)
(cherry picked from commit c1ae1649d22b356f2bbb2293cb421eb155fdee14)
(cherry picked from commit 816ea504c92b141b34759faaf814698499fa0c75)

2 files changed, 42 insertions, 34 deletions

diff --git a/releasenotes/notes/redis-connect-retries-c9adfc81eb06a4ab.yaml b/releasenotes/notes/redis-connect-retries-c9adfc81eb06a4ab.yaml
new file mode 100644
index 0000000..bf61307
--- /dev/null
+++ b/releasenotes/notes/redis-connect-retries-c9adfc81eb06a4ab.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Redis driver retries actions for up to 15 times when met with error
+    connecting to Redis.
diff --git a/tooz/drivers/redis.py b/tooz/drivers/redis.py
index fe12e52..b04d956 100644
--- a/tooz/drivers/redis.py
+++ b/tooz/drivers/redis.py
@@ -39,7 +39,7 @@ from tooz import utils
 LOG = logging.getLogger(__name__)
 
 
-def _handle_failures(func=None, n_tries=15):
+def _handle_failures(n_tries=15):
 
     """Translates common redis exceptions into tooz exceptions.
 
@@ -48,34 +48,37 @@ def _handle_failures(func=None, n_tries=15):
     :param func: the function to act on
     :param n_tries: the number of retries
     """
-
-    if func is None:
-        return functools.partial(_handle_failures, n_tries=n_tries)
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        ntries = n_tries
-        while ntries > 1:
-            try:
-                return func(*args, **kwargs)
-            except exceptions.ConnectionError as e:
-                # retry ntries times and then raise a connection error
-                ntries -= 1
-                if ntries >= 1:
+    def inner(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            ntries = n_tries
+            while ntries:
+                try:
+                    return func(*args, **kwargs)
+                except exceptions.ConnectionError as e:
+                    # retry ntries times and then raise a connection error
+                    ntries -= 1
+                    if not ntries:
+                        LOG.debug(
+                            "Redis connection error, "
+                            "retry limit has been reached, aborting - %s", e
+                        )
+                        utils.raise_with_cause(
+                            coordination.ToozConnectionError,
+                            encodeutils.exception_to_unicode(e),
+                            cause=e)
+                    LOG.debug("Redis connection error, will retry - %s", e)
+
+                except (exceptions.TimeoutError) as e:
                     utils.raise_with_cause(coordination.ToozConnectionError,
                                            encodeutils.exception_to_unicode(e),
                                            cause=e)
-
-            except (exceptions.TimeoutError) as e:
-                utils.raise_with_cause(coordination.ToozConnectionError,
-                                       encodeutils.exception_to_unicode(e),
-                                       cause=e)
-            except exceptions.RedisError as e:
-                utils.raise_with_cause(tooz.ToozError,
-                                       encodeutils.exception_to_unicode(e),
-                                       cause=e)
-        return func(*args, **kwargs)
-    return wrapper
+                except exceptions.RedisError as e:
+                    utils.raise_with_cause(tooz.ToozError,
+                                           encodeutils.exception_to_unicode(e),
+                                           cause=e)
+        return wrapper
+    return inner
 
 
 class RedisLock(locking.Lock):
@@ -91,7 +94,7 @@ class RedisLock(locking.Lock):
         self._coord = coord
         self._client = client
 
-    @_handle_failures
+    @_handle_failures()
     def is_still_owner(self):
         lock_tok = self._lock.local.token
         if not lock_tok:
@@ -99,11 +102,11 @@ class RedisLock(locking.Lock):
         owner_tok = self._client.get(self.name)
         return owner_tok == lock_tok
 
-    @_handle_failures
+    @_handle_failures()
     def break_(self):
         return bool(self._client.delete(self.name))
 
-    @_handle_failures
+    @_handle_failures()
     def acquire(self, blocking=True, shared=False):
         if shared:
             raise tooz.NotImplemented
@@ -115,7 +118,7 @@ class RedisLock(locking.Lock):
                 self._coord._acquired_locks.add(self)
         return acquired
 
-    @_handle_failures
+    @_handle_failures()
     def release(self):
         with self._exclusive_access:
             try:
@@ -127,7 +130,7 @@ class RedisLock(locking.Lock):
                 self._coord._acquired_locks.discard(self)
             return True
 
-    @_handle_failures
+    @_handle_failures()
     def heartbeat(self):
         with self._exclusive_access:
             if self.acquired:
@@ -464,7 +467,7 @@ return 1
             return master_client
         return redis.StrictRedis(**kwargs)
 
-    @_handle_failures
+    @_handle_failures()
     def _start(self):
         super(RedisDriver, self)._start()
         try:
@@ -537,7 +540,7 @@ return 1
     def _decode_group_id(self, group_id):
         return utils.to_binary(group_id, encoding=self._encoding)
 
-    @_handle_failures
+    @_handle_failures()
     def heartbeat(self):
         beat_id = self._encode_beat_id(self._member_id)
         expiry_ms = max(0, int(self.membership_timeout * 1000.0))
@@ -552,7 +555,7 @@ return 1
                             exc_info=True)
         return min(self.lock_timeout, self.membership_timeout)
 
-    @_handle_failures
+    @_handle_failures()
     def _stop(self):
         while self._acquired_locks:
             lock = self._acquired_locks.pop()