MDEV-14529 - InnoDB rw-locks: optimize memory barriers

Change lock_word from lint to int32_t: the latter is my_atomic_* friendly type.

5 files changed, 29 insertions, 34 deletions

diff --git a/storage/innobase/include/sync0rw.h b/storage/innobase/include/sync0rw.h
index 888a32007ce..7e9e4e0deeb 100644
--- a/storage/innobase/include/sync0rw.h
+++ b/storage/innobase/include/sync0rw.h
@@ -501,8 +501,8 @@ bool
 rw_lock_lock_word_decr(
 /*===================*/
 	rw_lock_t*	lock,		/*!< in/out: rw-lock */
-	ulint		amount,		/*!< in: amount to decrement */
-	lint		threshold);	/*!< in: threshold of judgement */
+	int32_t		amount,		/*!< in: amount to decrement */
+	int32_t		threshold);	/*!< in: threshold of judgement */
 #ifdef UNIV_DEBUG
 /******************************************************************//**
 Checks if the thread has locked the rw-lock in the specified mode, with
@@ -571,7 +571,7 @@ struct rw_lock_t
 #endif /* UNIV_DEBUG */
 {
 	/** Holds the state of the lock. */
-	volatile lint	lock_word;
+	volatile int32_t	lock_word;
 
 	/** 1: there are waiters */
 	volatile uint32_t	waiters;
diff --git a/storage/innobase/include/sync0rw.ic b/storage/innobase/include/sync0rw.ic
index a048476d0e8..23d3ed1fe0c 100644
--- a/storage/innobase/include/sync0rw.ic
+++ b/storage/innobase/include/sync0rw.ic
@@ -77,7 +77,7 @@ rw_lock_get_writer(
 /*===============*/
 	const rw_lock_t*	lock)	/*!< in: rw-lock */
 {
-	lint lock_word = lock->lock_word;
+	int32_t lock_word = lock->lock_word;
 
 	ut_ad(lock_word <= X_LOCK_DECR);
 	if (lock_word > X_LOCK_HALF_DECR) {
@@ -109,7 +109,7 @@ rw_lock_get_reader_count(
 /*=====================*/
 	const rw_lock_t*	lock)	/*!< in: rw-lock */
 {
-	lint lock_word = lock->lock_word;
+	int32_t lock_word = lock->lock_word;
 	ut_ad(lock_word <= X_LOCK_DECR);
 
 	if (lock_word > X_LOCK_HALF_DECR) {
@@ -145,7 +145,7 @@ rw_lock_get_x_lock_count(
 /*=====================*/
 	const rw_lock_t*	lock)	/*!< in: rw-lock */
 {
-	lint lock_copy = lock->lock_word;
+	int32_t lock_copy = lock->lock_word;
 	ut_ad(lock_copy <= X_LOCK_DECR);
 
 	if (lock_copy == 0 || lock_copy == -X_LOCK_HALF_DECR) {
@@ -178,7 +178,7 @@ rw_lock_get_sx_lock_count(
 	const rw_lock_t*	lock)	/*!< in: rw-lock */
 {
 #ifdef UNIV_DEBUG
-	lint lock_copy = lock->lock_word;
+	int32_t lock_copy = lock->lock_word;
 
 	ut_ad(lock_copy <= X_LOCK_DECR);
 
@@ -208,17 +208,16 @@ bool
 rw_lock_lock_word_decr(
 /*===================*/
 	rw_lock_t*	lock,		/*!< in/out: rw-lock */
-	ulint		amount,		/*!< in: amount to decrement */
-	lint		threshold)	/*!< in: threshold of judgement */
+	int32_t		amount,		/*!< in: amount to decrement */
+	int32_t		threshold)	/*!< in: threshold of judgement */
 {
-	lint local_lock_word;
+	int32_t local_lock_word;
 
-	local_lock_word = my_atomic_loadlint_explicit(&lock->lock_word,
-						      MY_MEMORY_ORDER_RELAXED);
+	local_lock_word = my_atomic_load32_explicit(&lock->lock_word,
+						    MY_MEMORY_ORDER_RELAXED);
 	while (local_lock_word > threshold) {
-		if (my_atomic_caslint(&lock->lock_word,
-				      &local_lock_word,
-				      local_lock_word - amount)) {
+		if (my_atomic_cas32(&lock->lock_word, &local_lock_word,
+				    local_lock_word - amount)) {
 			return(true);
 		}
 	}
@@ -306,9 +305,9 @@ rw_lock_x_lock_func_nowait(
 	const char*	file_name,/*!< in: file name where lock requested */
 	unsigned	line)	/*!< in: line where requested */
 {
-	lint oldval = X_LOCK_DECR;
+	int32_t oldval = X_LOCK_DECR;
 
-	if (my_atomic_caslint(&lock->lock_word, &oldval, 0)) {
+	if (my_atomic_cas32(&lock->lock_word, &oldval, 0)) {
 		lock->writer_thread = os_thread_get_curr_id();
 
 	} else if (os_thread_eq(lock->writer_thread, os_thread_get_curr_id())) {
@@ -357,7 +356,7 @@ rw_lock_s_unlock_func(
 	rw_lock_t*	lock)	/*!< in/out: rw-lock */
 {
 #ifdef UNIV_DEBUG
-	lint	dbg_lock_word = my_atomic_loadlint_explicit(
+	int32_t	dbg_lock_word = my_atomic_load32_explicit(
 			&lock->lock_word, MY_MEMORY_ORDER_RELAXED);
 	ut_ad(dbg_lock_word > -X_LOCK_DECR);
 	ut_ad(dbg_lock_word != 0);
@@ -367,7 +366,7 @@ rw_lock_s_unlock_func(
 	ut_d(rw_lock_remove_debug_info(lock, pass, RW_LOCK_S));
 
 	/* Increment lock_word to indicate 1 less reader */
-	lint	lock_word = my_atomic_addlint(&lock->lock_word, 1) + 1;
+	int32_t	lock_word = my_atomic_add32(&lock->lock_word, 1) + 1;
 	if (lock_word == 0 || lock_word == -X_LOCK_HALF_DECR) {
 
 		/* wait_ex waiter exists. It may not be asleep, but we signal
@@ -393,9 +392,9 @@ rw_lock_x_unlock_func(
 #endif /* UNIV_DEBUG */
 	rw_lock_t*	lock)	/*!< in/out: rw-lock */
 {
-	lint	lock_word;
-	lock_word = my_atomic_loadlint_explicit(&lock->lock_word,
-						MY_MEMORY_ORDER_RELAXED);
+	int32_t	lock_word;
+	lock_word = my_atomic_load32_explicit(&lock->lock_word,
+					      MY_MEMORY_ORDER_RELAXED);
 
 	ut_ad(lock_word == 0 || lock_word == -X_LOCK_HALF_DECR
 	      || lock_word <= -X_LOCK_DECR);
@@ -410,7 +409,7 @@ rw_lock_x_unlock_func(
 	if (lock_word == 0 || lock_word == -X_LOCK_HALF_DECR) {
 		/* There is 1 x-lock */
 		/* atomic increment is needed, because it is last */
-		if (my_atomic_addlint(&lock->lock_word, X_LOCK_DECR) <= -X_LOCK_DECR) {
+		if (my_atomic_add32(&lock->lock_word, X_LOCK_DECR) <= -X_LOCK_DECR) {
 			ut_error;
 		}
 
@@ -462,7 +461,7 @@ rw_lock_sx_unlock_func(
 		if (lock->lock_word > 0) {
 			lock->writer_thread = 0;
 
-			if (my_atomic_addlint(&lock->lock_word, X_LOCK_HALF_DECR) <= 0) {
+			if (my_atomic_add32(&lock->lock_word, X_LOCK_HALF_DECR) <= 0) {
 				ut_error;
 			}
 			/* Lock is now free. May have to signal read/write
diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h
index 1f8e245569e..136626b65d5 100644
--- a/storage/innobase/include/sync0types.h
+++ b/storage/innobase/include/sync0types.h
@@ -1163,15 +1163,11 @@ enum rw_lock_flag_t {
 #ifdef _WIN64
 #define my_atomic_addlint(A,B) my_atomic_add64((int64*) (A), (B))
 #define my_atomic_loadlint(A) my_atomic_load64((int64*) (A))
-#define my_atomic_loadlint_explicit(A,O) my_atomic_load64_explicit((int64*) (A), (O))
 #define my_atomic_storelint(A,B) my_atomic_store64((int64*) (A), (B))
-#define my_atomic_caslint(A,B,C) my_atomic_cas64((int64*) (A), (int64*) (B), (C))
 #else
 #define my_atomic_addlint my_atomic_addlong
 #define my_atomic_loadlint my_atomic_loadlong
-#define my_atomic_loadlint_explicit my_atomic_loadlong_explicit
 #define my_atomic_storelint my_atomic_storelong
-#define my_atomic_caslint my_atomic_caslong
 #endif
 
 /** Simple counter aligned to CACHE_LINE_SIZE
diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc
index f1589e1f3a7..c3c2324f554 100644
--- a/storage/innobase/sync/sync0arr.cc
+++ b/storage/innobase/sync/sync0arr.cc
@@ -589,7 +589,7 @@ sync_array_cell_print(
 			fprintf(file,
 				"number of readers " ULINTPF
 				", waiters flag %u, "
-				"lock_word: " ULINTPFx "\n"
+				"lock_word: %x\n"
 				"Last time read locked in file %s line %u\n"
 				"Last time write locked in file %s line %u"
 #if 0 /* JAN: TODO: FIX LATER */
diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc
index b7b68f98c19..14014a9425e 100644
--- a/storage/innobase/sync/sync0rw.cc
+++ b/storage/innobase/sync/sync0rw.cc
@@ -316,8 +316,8 @@ lock_loop:
 	/* Spin waiting for the writer field to become free */
 	HMT_low();
 	while (i < srv_n_spin_wait_rounds &&
-	       my_atomic_loadlint_explicit(&lock->lock_word,
-					   MY_MEMORY_ORDER_RELAXED) <= 0) {
+	       my_atomic_load32_explicit(&lock->lock_word,
+					 MY_MEMORY_ORDER_RELAXED) <= 0) {
 		if (srv_spin_wait_delay) {
 			ut_delay(ut_rnd_interval(0, srv_spin_wait_delay));
 		}
@@ -883,12 +883,12 @@ rw_lock_validate(
 /*=============*/
 	const rw_lock_t*	lock)	/*!< in: rw-lock */
 {
-	lint	lock_word;
+	int32_t	lock_word;
 
 	ut_ad(lock);
 
-	lock_word = my_atomic_loadlint_explicit(&lock->lock_word,
-						MY_MEMORY_ORDER_RELAXED);
+	lock_word = my_atomic_load32_explicit(const_cast<int32_t*>(&lock->lock_word),
+					      MY_MEMORY_ORDER_RELAXED);
 
 	ut_ad(lock->magic_n == RW_LOCK_MAGIC_N);
 	ut_ad(my_atomic_load32_explicit((int32*) &lock->waiters,