diff options
| author | Mitya_Eremeev <mitossvyaz@mail.ru> | 2022-05-23 21:53:01 +0300 |
|---|---|---|
| committer | Mitya_Eremeev <mitossvyaz@mail.ru> | 2022-05-25 20:21:42 +0300 |
| commit | ae5adcf3874d3ab40a8f6c475b450e72a75f5d23 (patch) | |
| tree | 84f6da6e9338434d8b3d30ed0a8ac511720851a9 | |
| parent | b21c20f79fbcf81b79706a5910ff802b405ca733 (diff) | |
| download | tooz-ae5adcf3874d3ab40a8f6c475b450e72a75f5d23.tar.gz | |
[etcd3gw] create new lease if expired.
It can happen membership lease can be expired
to the moment of lease refreshing.
In this case heartbeat will try to refresh
expired lease forever without success.
The patch checks if lease expired and create new one.
Closes-Bug: 1975524
Depends-On: https://review.opendev.org/c/openstack/etcd3gw/+/843003
Change-Id: I5017724fd0134f3f0a51f059640e641dd9853ff2
| -rw-r--r-- | releasenotes/notes/etcd3gw_create_new_lease_if_expired_during_refresh-1d631d36c21ea28c.yaml | 7 | ||||
| -rw-r--r-- | tooz/drivers/etcd3gw.py | 12 |
2 files changed, 17 insertions, 2 deletions
diff --git a/releasenotes/notes/etcd3gw_create_new_lease_if_expired_during_refresh-1d631d36c21ea28c.yaml b/releasenotes/notes/etcd3gw_create_new_lease_if_expired_during_refresh-1d631d36c21ea28c.yaml new file mode 100644 index 0000000..72360bb --- /dev/null +++ b/releasenotes/notes/etcd3gw_create_new_lease_if_expired_during_refresh-1d631d36c21ea28c.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + [etcd3gw] Membership lease can be already expired to the moment of lease + refreshing. In this case heartbeat will try to refresh expired lease + forever without success. The patch checks if lease is expired and creates + new one. diff --git a/tooz/drivers/etcd3gw.py b/tooz/drivers/etcd3gw.py index d4a7693..d426b74 100644 --- a/tooz/drivers/etcd3gw.py +++ b/tooz/drivers/etcd3gw.py @@ -14,6 +14,7 @@ import base64 import functools +import logging import threading import uuid @@ -27,6 +28,8 @@ from tooz import coordination from tooz import locking from tooz import utils +LOG = logging.getLogger(__name__) + def _encode(data): """Safely encode data for consumption of the gateway.""" @@ -218,6 +221,7 @@ class Etcd3Driver(coordination.CoordinationDriverWithExecutor): self.membership_timeout = int(options.get( 'membership_timeout', timeout)) self._acquired_locks = set() + self._membership_lease = None def _start(self): super(Etcd3Driver, self)._start() @@ -227,11 +231,15 @@ class Etcd3Driver(coordination.CoordinationDriverWithExecutor): return Etcd3Lock(self, name, self.lock_timeout) def heartbeat(self): + # TODO(kaifeng) use the same lease for locks? + if self._membership_lease.refresh() == -1: + expired_lease = self._membership_lease.id + self._membership_lease = self.client.lease(self.membership_timeout) + LOG.debug('Created new lease %s after previous lease %s expired.', + self._membership_lease.id, expired_lease) # NOTE(jaypipes): Copying because set can mutate during iteration for lock in self._acquired_locks.copy(): lock.heartbeat() - # TODO(kaifeng) use the same lease for locks? - self._membership_lease.refresh() return min(self.lock_timeout, self.membership_timeout) def watch_join_group(self, group_id, callback): |