1 files changed, 90 insertions, 144 deletions

diff --git a/storage/innobase/include/sync0rw.ic b/storage/innobase/include/sync0rw.ic
index a5a7cda14f9..eab89e2619e 100644
--- a/storage/innobase/include/sync0rw.ic
+++ b/storage/innobase/include/sync0rw.ic
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1995, 2009, Innobase Oy. All Rights Reserved.
+Copyright (c) 1995, 2011, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, Google Inc.
 
 Portions of this file contain modifications contributed and copyrighted by
@@ -18,8 +18,8 @@ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place, Suite 330, Boston, MA 02111-1307 USA
+this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
 
 *****************************************************************************/
 
@@ -90,7 +90,7 @@ rw_lock_set_waiter_flag(
 	rw_lock_t*	lock)	/*!< in/out: rw-lock */
 {
 #ifdef INNODB_RW_LOCKS_USE_ATOMICS
-	(void) os_compare_and_swap_ulint(&lock->waiters, 0, 1);
+	os_compare_and_swap_ulint(&lock->waiters, 0, 1);
 #else /* INNODB_RW_LOCKS_USE_ATOMICS */
 	lock->waiters = 1;
 #endif /* INNODB_RW_LOCKS_USE_ATOMICS */
@@ -107,7 +107,7 @@ rw_lock_reset_waiter_flag(
 	rw_lock_t*	lock)	/*!< in/out: rw-lock */
 {
 #ifdef INNODB_RW_LOCKS_USE_ATOMICS
-	(void) os_compare_and_swap_ulint(&lock->waiters, 1, 0);
+	os_compare_and_swap_ulint(&lock->waiters, 1, 0);
 #else /* INNODB_RW_LOCKS_USE_ATOMICS */
 	lock->waiters = 0;
 #endif /* INNODB_RW_LOCKS_USE_ATOMICS */
@@ -131,7 +131,7 @@ rw_lock_get_writer(
 	} else if (((-lock_word) % X_LOCK_DECR) == 0) {
 		return(RW_LOCK_EX);
 	} else {
-                ut_ad(lock_word > -X_LOCK_DECR);
+		ut_ad(lock_word > -X_LOCK_DECR);
 		return(RW_LOCK_WAIT_EX);
 	}
 }
@@ -200,7 +200,7 @@ rw_lock_lock_word_decr(
 	ulint		amount)		/*!< in: amount to decrement */
 {
 #ifdef INNODB_RW_LOCKS_USE_ATOMICS
-        lint local_lock_word = lock->lock_word;
+	lint local_lock_word = lock->lock_word;
 	while (local_lock_word > 0) {
 		if (os_compare_and_swap_lint(&lock->lock_word,
 					     local_lock_word,
@@ -244,7 +244,7 @@ rw_lock_lock_word_incr(
 
 	mutex_exit(&(lock->mutex));
 
-        return(local_lock_word);
+	return(local_lock_word);
 #endif /* INNODB_RW_LOCKS_USE_ATOMICS */
 }
 
@@ -308,7 +308,6 @@ rw_lock_s_lock_low(
 	const char*	file_name, /*!< in: file name where lock requested */
 	ulint		line)	/*!< in: line where requested */
 {
-	/* TODO: study performance of UNIV_LIKELY branch prediction hints. */
 	if (!rw_lock_lock_word_decr(lock, 1)) {
 		/* Locking did not succeed */
 		return(FALSE);
@@ -318,7 +317,7 @@ rw_lock_s_lock_low(
 	rw_lock_add_debug_info(lock, pass, RW_LOCK_SHARED, file_name, line);
 #endif
 	/* These debugging values are not set safely: they may be incorrect
-        or even refer to a line that is invalid for the file name. */
+	or even refer to a line that is invalid for the file name. */
 	lock->last_s_file_name = file_name;
 	lock->last_s_line = line;
 
@@ -409,7 +408,6 @@ rw_lock_s_lock_func(
 	ut_ad(!rw_lock_own(lock, RW_LOCK_EX));
 #endif /* UNIV_SYNC_DEBUG */
 
-	/* TODO: study performance of UNIV_LIKELY branch prediction hints. */
 	if (rw_lock_s_lock_low(lock, pass, file_name, line)) {
 
 		return; /* Success */
@@ -462,8 +460,12 @@ rw_lock_x_lock_func_nowait(
 		there is an exclusive writer and this is the writer thread. */
 		lock->lock_word -= X_LOCK_DECR;
 
+		/* Recursive x-locks must be multiples of X_LOCK_DECR. */
 		ut_ad(((-lock->lock_word) % X_LOCK_DECR) == 0);
 
+		/* Watch for too many recursive locks */
+		ut_ad(lock->lock_word < 0);
+
 	} else {
 		/* Failure */
 		return(FALSE);
@@ -502,10 +504,10 @@ rw_lock_s_unlock_func(
 	if (rw_lock_lock_word_incr(lock, 1) == 0) {
 
 		/* wait_ex waiter exists. It may not be asleep, but we signal
-                anyway. We do not wake other waiters, because they can't
-                exist without wait_ex waiter and wait_ex waiter goes first.*/
+		anyway. We do not wake other waiters, because they can't
+		exist without wait_ex waiter and wait_ex waiter goes first.*/
 		os_event_set(lock->wait_ex_event);
-		sync_array_object_signalled(sync_primary_wait_array);
+		sync_array_object_signalled();
 
 	}
 
@@ -517,31 +519,6 @@ rw_lock_s_unlock_func(
 }
 
 /******************************************************************//**
-Releases a shared mode lock when we know there are no waiters and none
-else will access the lock during the time this function is executed. */
-UNIV_INLINE
-void
-rw_lock_s_unlock_direct(
-/*====================*/
-	rw_lock_t*	lock)	/*!< in/out: rw-lock */
-{
-	ut_ad(lock->lock_word < X_LOCK_DECR);
-
-#ifdef UNIV_SYNC_DEBUG
-	rw_lock_remove_debug_info(lock, 0, RW_LOCK_SHARED);
-#endif
-
-	/* Decrease reader count by incrementing lock_word */
-	lock->lock_word++;
-
-	ut_ad(!lock->waiters);
-	ut_ad(rw_lock_validate(lock));
-#ifdef UNIV_SYNC_PERF_STAT
-	rw_s_exit_count++;
-#endif
-}
-
-/******************************************************************//**
 Releases an exclusive mode lock. */
 UNIV_INLINE
 void
@@ -564,6 +541,8 @@ rw_lock_x_unlock_func(
 	if (lock->lock_word == 0) {
 		/* Last caller in a possible recursive chain. */
 		lock->recursive = FALSE;
+		UNIV_MEM_INVALID(&lock->writer_thread,
+				 sizeof lock->writer_thread);
 	}
 
 #ifdef UNIV_SYNC_DEBUG
@@ -572,12 +551,12 @@ rw_lock_x_unlock_func(
 
 	if (rw_lock_lock_word_incr(lock, X_LOCK_DECR) == X_LOCK_DECR) {
 		/* Lock is now free. May have to signal read/write waiters.
-                We do not need to signal wait_ex waiters, since they cannot
-                exist when there is a writer. */
+		We do not need to signal wait_ex waiters, since they cannot
+		exist when there is a writer. */
 		if (lock->waiters) {
 			rw_lock_reset_waiter_flag(lock);
 			os_event_set(lock->event);
-			sync_array_object_signalled(sync_primary_wait_array);
+			sync_array_object_signalled();
 		}
 	}
 
@@ -588,38 +567,6 @@ rw_lock_x_unlock_func(
 #endif
 }
 
-/******************************************************************//**
-Releases an exclusive mode lock when we know there are no waiters, and
-none else will access the lock during the time this function is executed. */
-UNIV_INLINE
-void
-rw_lock_x_unlock_direct(
-/*====================*/
-	rw_lock_t*	lock)	/*!< in/out: rw-lock */
-{
-	/* Reset the exclusive lock if this thread no longer has an x-mode
-	lock */
-
-	ut_ad((lock->lock_word % X_LOCK_DECR) == 0);
-
-#ifdef UNIV_SYNC_DEBUG
-	rw_lock_remove_debug_info(lock, 0, RW_LOCK_EX);
-#endif
-
-	if (lock->lock_word == 0) {
-		lock->recursive = FALSE;
-	}
-
-	lock->lock_word += X_LOCK_DECR;
-
-	ut_ad(!lock->waiters);
-	ut_ad(rw_lock_validate(lock));
-
-#ifdef UNIV_SYNC_PERF_STAT
-	rw_x_exit_count++;
-#endif
-}
-
 #ifdef UNIV_PFS_RWLOCK
 
 /******************************************************************//**
@@ -643,9 +590,7 @@ pfs_rw_lock_create_func(
 	ulint		cline)		/*!< in: file line where created */
 {
 	/* Initialize the rwlock for performance schema */
-	lock->pfs_psi = (PSI_server && PFS_IS_INSTRUMENTED(key))
-				? PSI_server->init_rwlock(key, lock)
-				: NULL;
+	lock->pfs_psi = PSI_CALL(init_rwlock)(key, lock);
 
 	/* The actual function to initialize an rwlock */
 	rw_lock_create_func(lock,
@@ -656,7 +601,7 @@ pfs_rw_lock_create_func(
 			    cmutex_name,
 # endif /* UNIV_DEBUG */
 			    cfile_name,
-		            cline);
+			    cline);
 }
 /******************************************************************//**
 Performance schema instrumented wrap function for rw_lock_x_lock_func()
@@ -672,24 +617,23 @@ pfs_rw_lock_x_lock_func(
 	const char*	file_name,/*!< in: file name where lock requested */
 	ulint		line)	/*!< in: line where requested */
 {
-	struct PSI_rwlock_locker*	locker = NULL;
-	PSI_rwlock_locker_state		state;
+	if (lock->pfs_psi != NULL)
+	{
+		PSI_rwlock_locker*	locker;
+		PSI_rwlock_locker_state	state;
 
-	/* Record the entry of rw x lock request in performance schema */
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		locker = PSI_server->get_thread_rwlock_locker(
-			&state, lock->pfs_psi, PSI_RWLOCK_WRITELOCK);
-
-		if (locker) {
-			PSI_server->start_rwlock_wrwait(locker,
-							file_name, line);
-		}
-	}
+		/* Record the entry of rw x lock request in performance schema */
+		locker = PSI_CALL(start_rwlock_wrwait)(
+			&state, lock->pfs_psi, PSI_RWLOCK_WRITELOCK, file_name, line);
 
-	rw_lock_x_lock_func(lock, pass, file_name, line);
+		rw_lock_x_lock_func(lock, pass, file_name, line);
 
-	if (locker) {
-		PSI_server->end_rwlock_wrwait(locker, 0);
+		if (locker != NULL)
+			PSI_CALL(end_rwlock_wrwait)(locker, 0);
+	}
+	else
+	{
+		rw_lock_x_lock_func(lock, pass, file_name, line);
 	}
 }
 /******************************************************************//**
@@ -707,25 +651,25 @@ pfs_rw_lock_x_lock_func_nowait(
 				requested */
 	ulint		line)	/*!< in: line where requested */
 {
-	struct PSI_rwlock_locker*	locker = NULL;
-	PSI_rwlock_locker_state		state;
 	ibool	ret;
 
-	/* Record the entry of rw x lock request in performance schema */
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		locker = PSI_server->get_thread_rwlock_locker(
-			&state, lock->pfs_psi, PSI_RWLOCK_WRITELOCK);
+	if (lock->pfs_psi != NULL)
+	{
+		PSI_rwlock_locker*	locker;
+		PSI_rwlock_locker_state		state;
 
-		if (locker) {
-			PSI_server->start_rwlock_wrwait(locker,
-							file_name, line);
-		}
-	}
+		/* Record the entry of rw x lock request in performance schema */
+		locker = PSI_CALL(start_rwlock_wrwait)(
+			&state, lock->pfs_psi, PSI_RWLOCK_WRITELOCK, file_name, line);
 
-	ret = rw_lock_x_lock_func_nowait(lock, file_name, line);
+		ret = rw_lock_x_lock_func_nowait(lock, file_name, line);
 
-	if (locker) {
-		PSI_server->end_rwlock_wrwait(locker, 0);
+		if (locker != NULL)
+			PSI_CALL(end_rwlock_wrwait)(locker, ret);
+	}
+	else
+	{
+		ret = rw_lock_x_lock_func_nowait(lock, file_name, line);
 	}
 
 	return(ret);
@@ -740,8 +684,9 @@ pfs_rw_lock_free_func(
 /*==================*/
 	rw_lock_t*	lock)	/*!< in: pointer to rw-lock */
 {
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		PSI_server->destroy_rwlock(lock->pfs_psi);
+	if (lock->pfs_psi != NULL)
+	{
+		PSI_CALL(destroy_rwlock)(lock->pfs_psi);
 		lock->pfs_psi = NULL;
 	}
 
@@ -763,24 +708,26 @@ pfs_rw_lock_s_lock_func(
 				requested */
 	ulint		line)	/*!< in: line where requested */
 {
-	struct PSI_rwlock_locker*	locker = NULL;
-	PSI_rwlock_locker_state		state;
-
-	/* Instrumented to inform we are aquiring a shared rwlock */
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		locker = PSI_server->get_thread_rwlock_locker(
-			&state, lock->pfs_psi, PSI_RWLOCK_READLOCK);
-		if (locker) {
-			PSI_server->start_rwlock_rdwait(locker,
-							file_name, line);
-		}
-	}
+	if (lock->pfs_psi != NULL)
+	{
+		PSI_rwlock_locker*	locker;
+		PSI_rwlock_locker_state	state;
 
-	rw_lock_s_lock_func(lock, pass, file_name, line);
+		/* Instrumented to inform we are aquiring a shared rwlock */
+		locker = PSI_CALL(start_rwlock_rdwait)(
+			&state, lock->pfs_psi, PSI_RWLOCK_READLOCK, file_name, line);
 
-	if (locker) {
-		PSI_server->end_rwlock_rdwait(locker, 0);
+		rw_lock_s_lock_func(lock, pass, file_name, line);
+
+		if (locker != NULL)
+			PSI_CALL(end_rwlock_rdwait)(locker, 0);
+	}
+	else
+	{
+		rw_lock_s_lock_func(lock, pass, file_name, line);
 	}
+
+	return;
 }
 /******************************************************************//**
 Performance schema instrumented wrap function for rw_lock_s_lock_func()
@@ -798,24 +745,25 @@ pfs_rw_lock_s_lock_low(
 	const char*	file_name, /*!< in: file name where lock requested */
 	ulint		line)	/*!< in: line where requested */
 {
-	struct PSI_rwlock_locker*	locker = NULL;
-	PSI_rwlock_locker_state		state;
 	ibool	ret;
 
-	/* Instrumented to inform we are aquiring a shared rwlock */
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		locker = PSI_server->get_thread_rwlock_locker(
-			&state, lock->pfs_psi, PSI_RWLOCK_READLOCK);
-		if (locker) {
-			PSI_server->start_rwlock_rdwait(locker,
-							file_name, line);
-		}
-	}
+	if (lock->pfs_psi != NULL)
+	{
+		PSI_rwlock_locker*	locker;
+		PSI_rwlock_locker_state	state;
+
+		/* Instrumented to inform we are aquiring a shared rwlock */
+		locker = PSI_CALL(start_rwlock_rdwait)(
+			&state, lock->pfs_psi, PSI_RWLOCK_READLOCK, file_name, line);
 
-	ret = rw_lock_s_lock_low(lock, pass, file_name, line);
+		ret = rw_lock_s_lock_low(lock, pass, file_name, line);
 
-	if (locker) {
-		PSI_server->end_rwlock_rdwait(locker, 0);
+		if (locker != NULL)
+			PSI_CALL(end_rwlock_rdwait)(locker, ret);
+	}
+	else
+	{
+		ret = rw_lock_s_lock_low(lock, pass, file_name, line);
 	}
 
 	return(ret);
@@ -837,9 +785,8 @@ pfs_rw_lock_x_unlock_func(
 	rw_lock_t*	lock)	/*!< in/out: rw-lock */
 {
 	/* Inform performance schema we are unlocking the lock */
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		PSI_server->unlock_rwlock(lock->pfs_psi);
-	}
+	if (lock->pfs_psi != NULL)
+		PSI_CALL(unlock_rwlock)(lock->pfs_psi);
 
 	rw_lock_x_unlock_func(
 #ifdef UNIV_SYNC_DEBUG
@@ -864,9 +811,8 @@ pfs_rw_lock_s_unlock_func(
 	rw_lock_t*	lock)	/*!< in/out: rw-lock */
 {
 	/* Inform performance schema we are unlocking the lock */
-	if (UNIV_LIKELY(PSI_server && lock->pfs_psi)) {
-		PSI_server->unlock_rwlock(lock->pfs_psi);
-	}
+	if (lock->pfs_psi != NULL)
+		PSI_CALL(unlock_rwlock)(lock->pfs_psi);
 
 	rw_lock_s_unlock_func(
 #ifdef UNIV_SYNC_DEBUG