From 8361151765cc5efd72ad18c5553f80aa440a1d83 Mon Sep 17 00:00:00 2001 From: Sujatha Sivakumar Date: Tue, 1 Mar 2016 12:29:51 +0530 Subject: Bug#20685029: SLAVE IO THREAD SHOULD STOP WHEN DISK IS FULL Bug#21753696: MAKE SHOW SLAVE STATUS NON BLOCKING IF IO THREAD WAITS FOR DISK SPACE Problem: ======== Currently SHOW SLAVE STATUS blocks if IO thread waits for disk space. This makes automation tools verifying server health block on taking relevant action. Finally this will create SHOW SLAVE STATUS piles. Analysis: ========= SHOW SLAVE STATUS hangs on mi->data_lock if relay log write is waiting for free disk space while holding mi->data_lock. mi->data_lock is needed to protect the format description event (mi->format_description_event) which is accessed by the clients running FLUSH LOGS and slave IO thread. Note relay log writes don't need to be protected by mi->data_lock, LOCK_log is used to protect relay log between IO and SQL thread (see MYSQL_BIN_LOG::append_event). The code takes mi->data_lock to protect mi->format_description_event during relay log rotate which might get triggered right after relay log write. Fix: ==== Release the data_lock just for the duration of writing into relay log. Made change to ensure the following lock order is maintained to avoid deadlocks. data_lock, LOCK_log data_lock is held during relay log rotations to protect the description event. --- mysys/errors.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'mysys/errors.c') diff --git a/mysys/errors.c b/mysys/errors.c index ddd65836b30..942fd3230c0 100644 --- a/mysys/errors.c +++ b/mysys/errors.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -109,6 +109,7 @@ void init_glob_errs() */ void wait_for_free_space(const char *filename, int errors) { + size_t time_to_sleep= MY_WAIT_FOR_USER_TO_FIX_PANIC; if (!(errors % MY_WAIT_GIVE_USER_A_MESSAGE)) { my_printf_warning(EE(EE_DISK_FULL), @@ -119,10 +120,14 @@ void wait_for_free_space(const char *filename, int errors) } DBUG_EXECUTE_IF("simulate_no_free_space_error", { - (void) sleep(1); - return; + time_to_sleep= 1; }); - (void) sleep(MY_WAIT_FOR_USER_TO_FIX_PANIC); + DBUG_EXECUTE_IF("simulate_io_thd_wait_for_disk_space", + { + time_to_sleep= 1; + }); + + (void) sleep(time_to_sleep); } const char **get_global_errmsgs() -- cgit v1.2.1 From 3a8f43bec76d3d93a809b6a3c76e26e946ba0425 Mon Sep 17 00:00:00 2001 From: Sujatha Sivakumar Date: Mon, 11 Apr 2016 11:41:47 +0530 Subject: Bug#22897202: RPL_IO_THD_WAIT_FOR_DISK_SPACE HAS OCCASIONAL FAILURES Analysis: ========= Test script is not ensuring that "assert_grep.inc" should be called only after 'Disk is full' error is written to the error log. Test checks for "Queueing master event to the relay log" state. But this state is set before invoking 'queue_event'. Actual 'Disk is full' error happens at a very lower level. It can happen that we might even reset the debug point before even the actual disk full simulation occurs and the "Disk is full" message will never appear in the error log. In order to guarentee that we must have some mechanism where in after we write "Disk is full" error messge into the error log we must signal the test to execute SSS and then reset the debug point. So that test is deterministic. Fix: === Added debug sync point to make script deterministic. --- mysys/errors.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mysys/errors.c') diff --git a/mysys/errors.c b/mysys/errors.c index 942fd3230c0..b6064460535 100644 --- a/mysys/errors.c +++ b/mysys/errors.c @@ -15,7 +15,7 @@ #include "mysys_priv.h" #include "mysys_err.h" - +#include "m_string.h" #ifndef SHARED_LIBRARY const char *globerrs[GLOBERRS]= @@ -128,6 +128,7 @@ void wait_for_free_space(const char *filename, int errors) }); (void) sleep(time_to_sleep); + DEBUG_SYNC_C("disk_full_reached"); } const char **get_global_errmsgs() -- cgit v1.2.1 From ef3f09f0c9e62ea1bf86b33b5d97e954b3ae34fe Mon Sep 17 00:00:00 2001 From: Sujatha Sivakumar Date: Fri, 13 May 2016 16:42:45 +0530 Subject: Bug#23251517: SEMISYNC REPLICATION HANGING Revert following bug fix: Bug#20685029: SLAVE IO THREAD SHOULD STOP WHEN DISK IS FULL Bug#21753696: MAKE SHOW SLAVE STATUS NON BLOCKING IF IO THREAD WAITS FOR DISK SPACE This fix results in a deadlock between slave IO thread and SQL thread. (cherry picked from commit e3fea6c6dbb36c6ab21c4ab777224560e9608b53) --- mysys/errors.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'mysys/errors.c') diff --git a/mysys/errors.c b/mysys/errors.c index b6064460535..a6e2e300a1f 100644 --- a/mysys/errors.c +++ b/mysys/errors.c @@ -15,7 +15,7 @@ #include "mysys_priv.h" #include "mysys_err.h" -#include "m_string.h" + #ifndef SHARED_LIBRARY const char *globerrs[GLOBERRS]= @@ -109,7 +109,6 @@ void init_glob_errs() */ void wait_for_free_space(const char *filename, int errors) { - size_t time_to_sleep= MY_WAIT_FOR_USER_TO_FIX_PANIC; if (!(errors % MY_WAIT_GIVE_USER_A_MESSAGE)) { my_printf_warning(EE(EE_DISK_FULL), @@ -120,15 +119,10 @@ void wait_for_free_space(const char *filename, int errors) } DBUG_EXECUTE_IF("simulate_no_free_space_error", { - time_to_sleep= 1; - }); - DBUG_EXECUTE_IF("simulate_io_thd_wait_for_disk_space", - { - time_to_sleep= 1; + (void) sleep(1); + return; }); - - (void) sleep(time_to_sleep); - DEBUG_SYNC_C("disk_full_reached"); + (void) sleep(MY_WAIT_FOR_USER_TO_FIX_PANIC); } const char **get_global_errmsgs() -- cgit v1.2.1