From 1b3dc66e3117a09c95a00be2f649b975fdb25e2e Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 28 Jun 2013 15:19:30 +0200 Subject: MDEV-4506: Parallel replication: Intermediate commit. First step of splitting out part of Relay_log_info, so that different event groups being applied in parallel can each use their own copy. --- sql/sql_binlog.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'sql/sql_binlog.cc') diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 3bb5deab406..bef9a4c3475 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -44,6 +44,7 @@ void mysql_client_binlog_statement(THD* thd) { + struct rpl_group_info *rgi; DBUG_ENTER("mysql_client_binlog_statement"); DBUG_PRINT("info",("binlog base64: '%*s'", (int) (thd->lex->comment.length < 2048 ? @@ -196,6 +197,17 @@ void mysql_client_binlog_statement(THD* thd) } } + if (!(rgi= rli->group_info)) + { + if (!(rgi= rli->group_info= (struct rpl_group_info *) + my_malloc(sizeof(*rgi), MYF(0)))) + { + my_error(ER_OUTOFMEMORY, MYF(0), sizeof(*rgi)); + goto end; + } + bzero(rgi, sizeof(*rgi)); + } + rgi->rli= rli; ev= Log_event::read_log_event(bufptr, event_len, &error, rli->relay_log.description_event_for_exec, 0); @@ -232,7 +244,7 @@ void mysql_client_binlog_statement(THD* thd) (ev->flags & LOG_EVENT_SKIP_REPLICATION_F ? OPTION_SKIP_REPLICATION : 0); - err= ev->apply_event(rli); + err= ev->apply_event(rgi); thd->variables.option_bits= (thd->variables.option_bits & ~OPTION_SKIP_REPLICATION) | -- cgit v1.2.1 From 6d5f237e091ca7aa4fdd52c186af11fffc80b1c2 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 9 Jul 2013 13:15:53 +0200 Subject: MDEV-4506: Parallel replication: Intermediate commit. Fix a number of failures in the test suite. --- sql/sql_binlog.cc | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) (limited to 'sql/sql_binlog.cc') diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index bef9a4c3475..df6aab88200 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -44,7 +44,6 @@ void mysql_client_binlog_statement(THD* thd) { - struct rpl_group_info *rgi; DBUG_ENTER("mysql_client_binlog_statement"); DBUG_PRINT("info",("binlog base64: '%*s'", (int) (thd->lex->comment.length < 2048 ? @@ -100,6 +99,7 @@ void mysql_client_binlog_statement(THD* thd) const char *error= 0; char *buf= (char *) my_malloc(decoded_len, MYF(MY_WME)); Log_event *ev = 0; + struct rpl_group_info rgi(rli); /* Out of memory check @@ -197,17 +197,8 @@ void mysql_client_binlog_statement(THD* thd) } } - if (!(rgi= rli->group_info)) - { - if (!(rgi= rli->group_info= (struct rpl_group_info *) - my_malloc(sizeof(*rgi), MYF(0)))) - { - my_error(ER_OUTOFMEMORY, MYF(0), sizeof(*rgi)); - goto end; - } - bzero(rgi, sizeof(*rgi)); - } - rgi->rli= rli; + rgi.rli= rli; + rgi.thd= thd; ev= Log_event::read_log_event(bufptr, event_len, &error, rli->relay_log.description_event_for_exec, 0); @@ -244,7 +235,7 @@ void mysql_client_binlog_statement(THD* thd) (ev->flags & LOG_EVENT_SKIP_REPLICATION_F ? OPTION_SKIP_REPLICATION : 0); - err= ev->apply_event(rgi); + err= ev->apply_event(&rgi); thd->variables.option_bits= (thd->variables.option_bits & ~OPTION_SKIP_REPLICATION) | -- cgit v1.2.1 From ba4b937af2e3c9118071b1279bc39b6febca73a9 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 12 Jul 2013 14:36:20 +0200 Subject: MDEV-4506: Parallel replication: Intermediate commit Move the deferred event stuff from Relay_log_info to rpl_group_info to make it thread safe for parallel replication. --- sql/sql_binlog.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'sql/sql_binlog.cc') diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index df6aab88200..1b6713f1bc3 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -80,6 +80,8 @@ void mysql_client_binlog_statement(THD* thd) my_bool have_fd_event= TRUE; int err; Relay_log_info *rli; + struct rpl_group_info *rgi; + rli= thd->rli_fake; if (!rli) { @@ -95,11 +97,12 @@ void mysql_client_binlog_statement(THD* thd) new Format_description_log_event(4); have_fd_event= FALSE; } + if (!(rgi= thd->rgi_fake)) + rgi= thd->rgi_fake= new rpl_group_info(rli); const char *error= 0; char *buf= (char *) my_malloc(decoded_len, MYF(MY_WME)); Log_event *ev = 0; - struct rpl_group_info rgi(rli); /* Out of memory check @@ -197,8 +200,8 @@ void mysql_client_binlog_statement(THD* thd) } } - rgi.rli= rli; - rgi.thd= thd; + rgi->rli= rli; + rgi->thd= thd; ev= Log_event::read_log_event(bufptr, event_len, &error, rli->relay_log.description_event_for_exec, 0); @@ -235,7 +238,7 @@ void mysql_client_binlog_statement(THD* thd) (ev->flags & LOG_EVENT_SKIP_REPLICATION_F ? OPTION_SKIP_REPLICATION : 0); - err= ev->apply_event(&rgi); + err= ev->apply_event(rgi); thd->variables.option_bits= (thd->variables.option_bits & ~OPTION_SKIP_REPLICATION) | -- cgit v1.2.1 From d107bdaa01ad34b1afb4542b981b9b19af499f7b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2013 15:09:57 +0200 Subject: MDEV-4506, parallel replication. Some after-review fixes. --- sql/sql_binlog.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sql/sql_binlog.cc') diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 1b6713f1bc3..04cb4adcb2c 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -80,7 +80,7 @@ void mysql_client_binlog_statement(THD* thd) my_bool have_fd_event= TRUE; int err; Relay_log_info *rli; - struct rpl_group_info *rgi; + rpl_group_info *rgi; rli= thd->rli_fake; if (!rli) -- cgit v1.2.1 From 5633dd822711a269098bdb127c76c4b1250fcf8d Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 16 Sep 2013 14:33:49 +0200 Subject: MDEV-4506: parallel replication. Add a simple test case. Fix bugs found. --- sql/sql_binlog.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sql/sql_binlog.cc') diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 04cb4adcb2c..4f41b942345 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -273,7 +273,7 @@ void mysql_client_binlog_statement(THD* thd) end: thd->variables.option_bits= thd_options; - rli->slave_close_thread_tables(thd); + rgi->slave_close_thread_tables(thd); my_free(buf); DBUG_VOID_RETURN; } -- cgit v1.2.1 From 2e100cc5a493b6a0f6f907e0483a734c7fee2087 Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Mon, 14 Oct 2013 00:24:05 +0300 Subject: Fixes for parallel slave: - Made slaves temporary table multi-thread slave safe by adding mutex around save_temporary_table usage. - rli->save_temporary_tables is the active list of all used temporary tables - This is copied to THD->temporary_tables when temporary tables are opened and updated when temporary tables are closed - Added THD->lock_temporary_tables() and THD->unlock_temporary_tables() to simplify this. - Relay_log_info->sql_thd renamed to Relay_log_info->sql_driver_thd to avoid wrong usage for merged code. - Added is_part_of_group() to mark functions that are part of the next function. This replaces setting IN_STMT when events are executed. - Added is_begin(), is_commit() and is_rollback() functions to Query_log_event to simplify code. - If slave_skip_counter is set run things in single threaded mode. This simplifies code for skipping events. - Updating state of relay log (IN_STMT and IN_TRANSACTION) is moved to one single function: update_state_of_relay_log() We can't use OPTION_BEGIN to check for the state anymore as the sql_driver and sql execution threads may be different. Clear IN_STMT and IN_TRANSACTION in init_relay_log_pos() and Relay_log_info::cleanup_context() to ensure the flags doesn't survive slave restarts is_in_group() is now independent of state of executed transaction. - Reset thd->transaction.all.modified_non_trans_table() if we did set it for single table row events. This was mainly for keeping the flag as documented. - Changed slave_open_temp_tables to uint32 to be able to use atomic operators on it. - Relay_log_info::sleep_lock -> rpl_group_info::sleep_lock - Relay_log_info::sleep_cond -> rpl_group_info::sleep_cond - Changed some functions to take rpl_group_info instead of Relay_log_info to make them multi-slave safe and to simplify usage - do_shall_skip() - continue_group() - sql_slave_killed() - next_event() - Simplifed arguments to io_salve_killed(), check_io_slave_killed() and sql_slave_killed(); No reason to supply THD as this is part of the given structure. - set_thd_in_use_temporary_tables() removed as in_use is set on usage - Added information to thd_proc_info() which thread is waiting for slave mutex to exit. - In open_table() reuse code from find_temporary_table() Other things: - More DBUG statements - Fixed the rpl_incident.test can be run with --debug - More comments - Disabled not used function rpl_connect_master() mysql-test/suite/perfschema/r/all_instances.result: Moved sleep_lock and sleep_cond to rpl_group_info mysql-test/suite/rpl/r/rpl_incident.result: Updated result mysql-test/suite/rpl/t/rpl_incident-master.opt: Not needed anymore mysql-test/suite/rpl/t/rpl_incident.test: Fixed that test can be run with --debug sql/handler.cc: More DBUG_PRINT sql/log.cc: More comments sql/log_event.cc: Added DBUG statements do_shall_skip(), continue_group() now takes rpl_group_info param Use is_begin(), is_commit() and is_rollback() functions instead of inspecting query string We don't have set slaves temporary tables 'in_use' as this is now done when tables are opened. Removed IN_STMT flag setting. This is now done in update_state_of_relay_log() Use IN_TRANSACTION flag to test state of relay log. In rows_event_stmt_cleanup() reset thd->transaction.all.modified_non_trans_table if we had set this before. sql/log_event.h: do_shall_skip(), continue_group() now takes rpl_group_info param Added is_part_of_group() to mark events that are part of the next event. This replaces setting IN_STMT when events are executed. Added is_begin(), is_commit() and is_rollback() functions to Query_log_event to simplify code. sql/log_event_old.cc: Removed IN_STMT flag setting. This is now done in update_state_of_relay_log() do_shall_skip(), continue_group() now takes rpl_group_info param sql/log_event_old.h: Added is_part_of_group() to mark events that are part of the next event. do_shall_skip(), continue_group() now takes rpl_group_info param sql/mysqld.cc: Changed slave_open_temp_tables to uint32 to be able to use atomic operators on it. Relay_log_info::sleep_lock -> Rpl_group_info::sleep_lock Relay_log_info::sleep_cond -> Rpl_group_info::sleep_cond sql/mysqld.h: Updated types and names sql/rpl_gtid.cc: More DBUG sql/rpl_parallel.cc: Updated TODO section Set thd for event that is execution Use new is_begin(), is_commit() and is_rollback() functions. More comments sql/rpl_rli.cc: sql_thd -> sql_driver_thd Relay_log_info::sleep_lock -> rpl_group_info::sleep_lock Relay_log_info::sleep_cond -> rpl_group_info::sleep_cond Clear IN_STMT and IN_TRANSACTION in init_relay_log_pos() and Relay_log_info::cleanup_context() to ensure the flags doesn't survive slave restarts. Reset table->in_use for temporary tables as the table may have been used by another THD. Use IN_TRANSACTION instead of OPTION_BEGIN to check state of relay log. Removed IN_STMT flag setting. This is now done in update_state_of_relay_log() sql/rpl_rli.h: Changed relay log state flags to bit masks instead of bit positions (most other code we have uses bit masks) Added IN_TRANSACTION to mark if we are in a BEGIN ... COMMIT section. save_temporary_tables is now thread safe Relay_log_info::sleep_lock -> rpl_group_info::sleep_lock Relay_log_info::sleep_cond -> rpl_group_info::sleep_cond Relay_log_info->sql_thd renamed to Relay_log_info->sql_driver_thd to avoid wrong usage for merged code is_in_group() is now independent of state of executed transaction. sql/slave.cc: Simplifed arguments to io_salve_killed(), sql_slave_killed() and check_io_slave_killed(); No reason to supply THD as this is part of the given structure. set_thd_in_use_temporary_tables() removed as in_use is set on usage in sql_base.cc sql_thd -> sql_driver_thd More DBUG Added update_state_of_relay_log() which will calculate the IN_STMT and IN_TRANSACTION state of the relay log after the current element is executed. If slave_skip_counter is set run things in single threaded mode. Simplifed arguments to io_salve_killed(), check_io_slave_killed() and sql_slave_killed(); No reason to supply THD as this is part of the given structure. Added information to thd_proc_info() which thread is waiting for slave mutex to exit. Disabled not used function rpl_connect_master() Updated argument to next_event() sql/sql_base.cc: Added mutex around usage of slave's temporary tables. The active list is always kept up to date in sql->rgi_slave->save_temporary_tables. Clear thd->temporary_tables after query (safety) More DBUG When using temporary table, set table->in_use to current thd as the THD may be different for slave threads. Some code is ifdef:ed with REMOVE_AFTER_MERGE_WITH_10 as the given code in 10.0 is not yet in this tree. In open_table() reuse code from find_temporary_table() sql/sql_binlog.cc: rli->sql_thd -> rli->sql_driver_thd Remove duplicate setting of rgi->rli sql/sql_class.cc: Added helper functions rgi_lock_temporary_tables() and rgi_unlock_temporary_tables() Would have been nicer to have these inline, but there was no easy way to do that sql/sql_class.h: Added functions to protect slaves temporary tables sql/sql_parse.cc: Added DBUG_PRINT sql/transaction.cc: Added comment --- sql/sql_binlog.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'sql/sql_binlog.cc') diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 4f41b942345..9bcfe64cf2d 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -99,6 +99,7 @@ void mysql_client_binlog_statement(THD* thd) } if (!(rgi= thd->rgi_fake)) rgi= thd->rgi_fake= new rpl_group_info(rli); + rgi->thd= thd; const char *error= 0; char *buf= (char *) my_malloc(decoded_len, MYF(MY_WME)); @@ -115,7 +116,7 @@ void mysql_client_binlog_statement(THD* thd) goto end; } - rli->sql_thd= thd; + rli->sql_driver_thd= thd; rli->no_storage= TRUE; for (char const *strptr= thd->lex->comment.str ; @@ -200,8 +201,6 @@ void mysql_client_binlog_statement(THD* thd) } } - rgi->rli= rli; - rgi->thd= thd; ev= Log_event::read_log_event(bufptr, event_len, &error, rli->relay_log.description_event_for_exec, 0); -- cgit v1.2.1