ctdb-recoverd: Add handler for lost recovery lock

If the process holding the recovery lock terminates unexpectedly then
the recovery daemon needs to know that the lock is no longer held.

While here, rename hold_reclock_handler() to take_reclock_handler() so
there is a clear difference between the two handler names.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

1 files changed, 19 insertions, 2 deletions

diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index 34b506a27d0..1becc53f324 100644
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -1553,7 +1553,7 @@ struct hold_reclock_state {
 	double latency;
 };
 
-static void hold_reclock_handler(char status,
+static void take_reclock_handler(char status,
 				 double latency,
 				 void *private_data)
 {
@@ -1578,6 +1578,22 @@ static void hold_reclock_handler(char status,
 	s->locked = (status == '0') ;
 }
 
+static bool ctdb_recovery_lock(struct ctdb_recoverd *rec);
+
+static void lost_reclock_handler(void *private_data)
+{
+	struct ctdb_recoverd *rec = talloc_get_type_abort(
+		private_data, struct ctdb_recoverd);
+
+	DEBUG(DEBUG_ERR,
+	      ("Recovery lock helper terminated unexpectedly - "
+	       "trying to retake recovery lock\n"));
+	TALLOC_FREE(rec->recovery_lock_handle);
+	if (! ctdb_recovery_lock(rec)) {
+		DEBUG(DEBUG_ERR, ("Failed to take recovery lock\n"));
+	}
+}
+
 static bool ctdb_recovery_lock(struct ctdb_recoverd *rec)
 {
 	struct ctdb_context *ctdb = rec->ctdb;
@@ -1589,7 +1605,8 @@ static bool ctdb_recovery_lock(struct ctdb_recoverd *rec)
 	};
 
 	h = ctdb_cluster_mutex(rec, ctdb, ctdb->recovery_lock, 0,
-			       hold_reclock_handler, &s, NULL, NULL);
+			       take_reclock_handler, &s,
+			       lost_reclock_handler, rec);
 	if (h == NULL) {
 		return false;
 	}