diff options
| -rw-r--r-- | mysql-test/include/show_gtid_list.inc | 15 | ||||
| -rw-r--r-- | mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result | 78 | ||||
| -rw-r--r-- | mysql-test/suite/binlog/r/binlog_gtid_delete_domain_debug.result | 6 | ||||
| -rw-r--r-- | mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test | 137 | ||||
| -rw-r--r-- | mysql-test/suite/binlog/t/binlog_gtid_delete_domain_debug.test | 11 | ||||
| -rw-r--r-- | mysql-test/suite/perfschema/r/start_server_low_digest.result | 4 | ||||
| -rw-r--r-- | mysql-test/suite/rpl/r/rpl_gtid_delete_domain.result | 30 | ||||
| -rw-r--r-- | mysql-test/suite/rpl/t/rpl_gtid_delete_domain.test | 95 | ||||
| -rw-r--r-- | sql/lex.h | 1 | ||||
| -rw-r--r-- | sql/log.cc | 195 | ||||
| -rw-r--r-- | sql/log.h | 7 | ||||
| -rw-r--r-- | sql/rpl_gtid.cc | 151 | ||||
| -rw-r--r-- | sql/rpl_gtid.h | 9 | ||||
| -rw-r--r-- | sql/share/errmsg-utf8.txt | 2 | ||||
| -rw-r--r-- | sql/sql_lex.cc | 5 | ||||
| -rw-r--r-- | sql/sql_lex.h | 7 | ||||
| -rw-r--r-- | sql/sql_reload.cc | 5 | ||||
| -rw-r--r-- | sql/sql_repl.cc | 68 | ||||
| -rw-r--r-- | sql/sql_repl.h | 1 | ||||
| -rw-r--r-- | sql/sql_yacc.yy | 22 |
20 files changed, 772 insertions, 77 deletions
diff --git a/mysql-test/include/show_gtid_list.inc b/mysql-test/include/show_gtid_list.inc new file mode 100644 index 00000000000..96f813f180c --- /dev/null +++ b/mysql-test/include/show_gtid_list.inc @@ -0,0 +1,15 @@ +# ==== Purpose ==== +# +# Extract Gtid_list info from SHOW BINLOG EVENTS output masking +# non-deterministic fields. +# +# ==== Usage ==== +# +# [--let $binlog_file=filename +# +if ($binlog_file) +{ + --let $_in_binlog_file=in '$binlog_file' +} +--replace_column 2 # 5 # +--eval show binlog events $_in_binlog_file limit 1,1 diff --git a/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result b/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result new file mode 100644 index 00000000000..daf2f4d6490 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_flush_binlogs_delete_domain.result @@ -0,0 +1,78 @@ +RESET MASTER; +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (); +and the command execution is effective thence rotates binlog as usual +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +Non-existed domain is warned, the command completes without rotation +but with a warning +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (99); +Warnings: +Warning 1982 The gtid domain being deleted ('99') is not in the current binlog state +show binary logs; +Log_name File_size +master-bin.000001 # +master-bin.000002 # +SET @@SESSION.gtid_domain_id=1; +SET @@SESSION.server_id=1; +CREATE TABLE t (a int); +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); +ERROR HY000: Could not delete gtid domain. Reason: binlog files may contain gtids from the domain ('1') being deleted. Make sure to first purge those files. +FLUSH BINARY LOGS; +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); +ERROR HY000: Could not delete gtid domain. Reason: binlog files may contain gtids from the domain ('1') being deleted. Make sure to first purge those files. +PURGE BINARY LOGS TO 'master-bin.000003';; +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); +Gtid_list of the current binlog does not contain '1': +show binlog events in 'master-bin.000004' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000004 # Gtid_list 1 # [] +But the previous log's Gtid_list may have it which explains a warning from the following command +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-1-1] +Already deleted domain in Gtid_list of the earliest log is benign +but may cause a warning +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); +Warnings: +Warning 1982 The current gtid binlog state is incompatible with a former one missing gtids from the '1-1' domain-server pair which is referred to in the gtid list describing an earlier state. Ignore if the domain ('1') was already explicitly deleted. +Warning 1982 The gtid domain being deleted ('1') is not in the current binlog state +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0); +ERROR HY000: Could not delete gtid domain. Reason: binlog files may contain gtids from the domain ('1') being deleted. Make sure to first purge those files. +FLUSH BINARY LOGS; +PURGE BINARY LOGS TO 'master-bin.000005'; +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0); +Warnings: +Warning 1982 The gtid domain being deleted ('0') is not in the current binlog state +Gtid_list of the current binlog does not contain 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0: +show binlog events in 'master-bin.000006' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000006 # Gtid_list 1 # [] +SET @@SESSION.gtid_domain_id=1;; +SET @@SESSION.server_id=1; +SET @@SESSION.gtid_seq_no=1; +INSERT INTO t SET a=1; +SET @@SESSION.server_id=2; +SET @@SESSION.gtid_seq_no=2; +INSERT INTO t SET a=2; +SET @@SESSION.gtid_domain_id=11; +SET @@SESSION.server_id=11; +SET @@SESSION.gtid_seq_no=11; +INSERT INTO t SET a=11; +SET @gtid_binlog_state_saved=@@GLOBAL.gtid_binlog_state; +FLUSH BINARY LOGS; +SET @@SESSION.gtid_domain_id=11; +SET @@SESSION.server_id=11; +SET @@SESSION.gtid_seq_no=1; +INSERT INTO t SET a=1; +SELECT @gtid_binlog_state_saved "as original state", @@GLOBAL.gtid_binlog_state as "out of order for 11 domain state"; +as original state out of order for 11 domain state +1-1-1,1-2-2,11-11-11 1-1-1,1-2-2,11-11-1 +PURGE BINARY LOGS TO 'master-bin.000007'; +the following command succeeds with warnings +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); +Warnings: +Warning 1982 The current gtid binlog state is incompatible with a former one having a gtid '11-11-1' which is less than the '11-11-11' of the gtid list describing an earlier state. The state may have been affected by manually injecting a lower sequence number gtid or via replication. +DROP TABLE t; +RESET MASTER; diff --git a/mysql-test/suite/binlog/r/binlog_gtid_delete_domain_debug.result b/mysql-test/suite/binlog/r/binlog_gtid_delete_domain_debug.result new file mode 100644 index 00000000000..5193267e2f2 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_gtid_delete_domain_debug.result @@ -0,0 +1,6 @@ +SET @@SESSION.debug_dbug='+d,inject_binlog_delete_domain_init_error'; +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (99); +ERROR HY000: Could not delete gtid domain. Reason: injected error. +SHOW WARNINGS; +Level Code Message +Error 1982 Could not delete gtid domain. Reason: injected error. diff --git a/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test b/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test new file mode 100644 index 00000000000..0faafa35a1b --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_flush_binlogs_delete_domain.test @@ -0,0 +1,137 @@ +# Prove basic properties of +# +# FLUSH BINARY LOGS DELETE_DOMAIN_ID = (...) +# +# The command removes the supplied list of domains from the current +# @@global.gtid_binlog_state provided the binlog files do not contain +# events from such domains. + +# The test is not format specific. One format is chosen to run it. +--source include/have_binlog_format_mixed.inc + +# Reset binlog state +RESET MASTER; + +# Empty list is accepted +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (); +--echo and the command execution is effective thence rotates binlog as usual +--source include/show_binary_logs.inc + +--echo Non-existed domain is warned, the command completes without rotation +--echo but with a warning +--let $binlog_pre_flush=query_get_value(SHOW MASTER STATUS, Position, 1) +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (99); +--let $binlog_start=$binlog_pre_flush +--source include/show_binary_logs.inc + +# Log one event in a specified domain and try to delete the domain +SET @@SESSION.gtid_domain_id=1; +SET @@SESSION.server_id=1; +CREATE TABLE t (a int); + +--error ER_BINLOG_CANT_DELETE_GTID_DOMAIN +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); + +# the same error after log rotation +FLUSH BINARY LOGS; +--error ER_BINLOG_CANT_DELETE_GTID_DOMAIN +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); + +# the latest binlog does not really contain any events incl ones from 1-domain +--let $purge_to_binlog= query_get_value(SHOW MASTER STATUS, File, 1) +--eval PURGE BINARY LOGS TO '$purge_to_binlog'; +# So now it's safe to delete +--error 0 +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); +--echo Gtid_list of the current binlog does not contain '1': +--let $binlog_file=query_get_value(SHOW MASTER STATUS, File, 1) +--source include/show_gtid_list.inc +--echo But the previous log's Gtid_list may have it which explains a warning from the following command +--let $binlog_file=$purge_to_binlog +--source include/show_gtid_list.inc + +--echo Already deleted domain in Gtid_list of the earliest log is benign +--echo but may cause a warning +--error 0 +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (1); + +# Few domains delete. The chosen number verifies among others how +# expected overrun of the static buffers of underlying dynamic arrays is doing. +--let $domain_cnt=17 +--let $server_in_domain_cnt=3 +--let $domain_list= +--disable_query_log +while ($domain_cnt) +{ + --let servers=$server_in_domain_cnt + --eval SET @@SESSION.gtid_domain_id=$domain_cnt + while ($servers) + { + --eval SET @@SESSION.server_id=10*$domain_cnt + $servers + --eval INSERT INTO t SET a=@@SESSION.server_id + + --dec $servers + } + --let $domain_list= $domain_cnt, $domain_list + + --dec $domain_cnt +} +--enable_query_log +--let $zero=0 +--let $domain_list= $domain_list$zero + +--error ER_BINLOG_CANT_DELETE_GTID_DOMAIN +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID = ($domain_list) + +# Now satisfy the safety condtion to purge log files containing $domain list +FLUSH BINARY LOGS; +--let $purge_to_binlog= query_get_value(SHOW MASTER STATUS, File, 1) +--eval PURGE BINARY LOGS TO '$purge_to_binlog' +--error 0 +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID = ($domain_list) +--echo Gtid_list of the current binlog does not contain $domain_list: +--let $binlog_file=query_get_value(SHOW MASTER STATUS, File, 1) +--source include/show_gtid_list.inc + +# Show reaction on @@global.gtid_binlog_state not succeeding +# earlier state as described by the 1st binlog' Gtid_list. +# Now let it be out-order gtid logged to a domain unrelated to deletion. + +--let $del_d_id=1 +--eval SET @@SESSION.gtid_domain_id=$del_d_id; +SET @@SESSION.server_id=1; +SET @@SESSION.gtid_seq_no=1; +INSERT INTO t SET a=1; +SET @@SESSION.server_id=2; +SET @@SESSION.gtid_seq_no=2; +INSERT INTO t SET a=2; + +SET @@SESSION.gtid_domain_id=11; +SET @@SESSION.server_id=11; +SET @@SESSION.gtid_seq_no=11; +INSERT INTO t SET a=11; + +SET @gtid_binlog_state_saved=@@GLOBAL.gtid_binlog_state; +FLUSH BINARY LOGS; + +# Inject out of order for domain '11' before +SET @@SESSION.gtid_domain_id=11; +SET @@SESSION.server_id=11; +SET @@SESSION.gtid_seq_no=1; +INSERT INTO t SET a=1; + +SELECT @gtid_binlog_state_saved "as original state", @@GLOBAL.gtid_binlog_state as "out of order for 11 domain state"; + +# to delete '1', first to purge logs containing its events +--let $purge_to_binlog= query_get_value(SHOW MASTER STATUS, File, 1) +--eval PURGE BINARY LOGS TO '$purge_to_binlog' + +--echo the following command succeeds with warnings +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID = ($del_d_id) + +# +# Cleanup +# + +DROP TABLE t; +RESET MASTER; diff --git a/mysql-test/suite/binlog/t/binlog_gtid_delete_domain_debug.test b/mysql-test/suite/binlog/t/binlog_gtid_delete_domain_debug.test new file mode 100644 index 00000000000..5de549c45bb --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_gtid_delete_domain_debug.test @@ -0,0 +1,11 @@ +# Check "internal" error branches of +# FLUSH BINARY LOGS DELETE_DOMAIN_ID = (...) +# handler. +--source include/have_debug.inc +--source include/have_binlog_format_mixed.inc + +SET @@SESSION.debug_dbug='+d,inject_binlog_delete_domain_init_error'; +--error ER_BINLOG_CANT_DELETE_GTID_DOMAIN +FLUSH BINARY LOGS DELETE_DOMAIN_ID = (99); + +SHOW WARNINGS; diff --git a/mysql-test/suite/perfschema/r/start_server_low_digest.result b/mysql-test/suite/perfschema/r/start_server_low_digest.result index 8cc92f21964..6fc41fbb715 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_digest.result +++ b/mysql-test/suite/perfschema/r/start_server_low_digest.result @@ -8,5 +8,5 @@ SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1 #################################### SELECT event_name, digest, digest_text, sql_text FROM events_statements_history_long; event_name digest digest_text sql_text -statement/sql/truncate e1c917a43f978456fab15240f89372ca TRUNCATE TABLE truncate table events_statements_history_long -statement/sql/select 3f7ca34376814d0e985337bd588b5ffd SELECT ? + ? + SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1 +statement/sql/truncate 6206ac02a54d832f55015e480e6f2213 TRUNCATE TABLE truncate table events_statements_history_long +statement/sql/select 4cc1c447d79877c4e8df0423fd0cde9a SELECT ? + ? + SELECT 1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1+1 diff --git a/mysql-test/suite/rpl/r/rpl_gtid_delete_domain.result b/mysql-test/suite/rpl/r/rpl_gtid_delete_domain.result new file mode 100644 index 00000000000..3558a6764d1 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_gtid_delete_domain.result @@ -0,0 +1,30 @@ +include/master-slave.inc +[connection master] +SET @@SESSION.gtid_domain_id=0; +CREATE TABLE t (a INT); +call mtr.add_suppression("connecting slave requested to start from.*which is not in the master's binlog"); +include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; +SET @@SESSION.gtid_domain_id=11; +SET @@SESSION.server_id=111; +SET @@SESSION.gtid_seq_no=1; +INSERT INTO t SET a=1; +SET @save.gtid_slave_pos=@@global.gtid_slave_pos; +SET @@global.gtid_slave_pos=concat(@@global.gtid_slave_pos, ",", 11, "-", 111, "-", 1 + 1); +Warnings: +Warning 1947 Specified GTID 0-1-1 conflicts with the binary log which contains a more recent GTID 0-2-2. If MASTER_GTID_POS=CURRENT_POS is used, the binlog position will override the new value of @@gtid_slave_pos. +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1236] +FLUSH BINARY LOGS; +PURGE BINARY LOGS TO 'master-bin.000002';; +FLUSH BINARY LOGS DELETE_DOMAIN_ID=(11); +include/start_slave.inc +INSERT INTO t SET a=1; +include/wait_for_slave_io_error.inc [errno=1236] +FLUSH BINARY LOGS; +PURGE BINARY LOGS TO 'master-bin.000004';; +FLUSH BINARY LOGS DELETE_DOMAIN_ID=(11); +include/start_slave.inc +SET @@SESSION.gtid_domain_id=0; +DROP TABLE t; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_delete_domain.test b/mysql-test/suite/rpl/t/rpl_gtid_delete_domain.test new file mode 100644 index 00000000000..5abedd7eb37 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_gtid_delete_domain.test @@ -0,0 +1,95 @@ +# In case master's gtid binlog state is divergent from the slave's gtid_slave_pos +# slave may not be able to connect. +# For instance when slave is more updated in some of domains, see +# MDEV-12012 as example, the master's state may require adjustment. +# In a specific case of an "old" divergent domain, that is there +# won't be no more event groups from it generated, the states can be +# made compatible with wiping the problematic domain away. After that slave +# becomes connectable. +# +# Notice that the slave applied gtid state is not really required to +# be similarly cleaned in order for replication to flow. +# However this could lead to an expected error when the master +# resumes binlogging of such domain which the test demonstrate. + +--source include/master-slave.inc + +--connection master +# enforce the default domain_id binlogging explicitly +SET @@SESSION.gtid_domain_id=0; +CREATE TABLE t (a INT); +--sync_slave_with_master + +--connection slave +call mtr.add_suppression("connecting slave requested to start from.*which is not in the master's binlog"); + +--source include/stop_slave.inc +CHANGE MASTER TO master_use_gtid=slave_pos; + +--connection master +# create extra gtid domains for binlog state +--let $extra_domain_id=11 +--let $extra_domain_server_id=111 +--let $extra_gtid_seq_no=1 +--eval SET @@SESSION.gtid_domain_id=$extra_domain_id +--eval SET @@SESSION.server_id=$extra_domain_server_id +--eval SET @@SESSION.gtid_seq_no=$extra_gtid_seq_no +INSERT INTO t SET a=1; + +# +# Set up the slave replication state as if slave knows more events from the extra +# domain. +# +--connection slave +SET @save.gtid_slave_pos=@@global.gtid_slave_pos; +--eval SET @@global.gtid_slave_pos=concat(@@global.gtid_slave_pos, ",", $extra_domain_id, "-", $extra_domain_server_id, "-", $extra_gtid_seq_no + 1) + +# unsuccessful attempt to start slave +START SLAVE IO_THREAD; +--let $slave_io_errno=1236 +--source include/wait_for_slave_io_error.inc + +--connection master +# adjust the master binlog state +FLUSH BINARY LOGS; +--let $purge_to_binlog= query_get_value(SHOW MASTER STATUS, File, 1) +--eval PURGE BINARY LOGS TO '$purge_to_binlog'; +# with final removal of the extra domain +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID=($extra_domain_id) + +--connection slave +# start the slave sucessfully +--source include/start_slave.inc + +--connection master +# but the following gtid from the *extra* domain will break replication +INSERT INTO t SET a=1; + +# take note of the slave io thread error due to being dismissed +# extra domain at connection to master which tried becoming active; +# slave is to stop. +--connection slave +--let $errno=1236 +--source include/wait_for_slave_io_error.inc + +# let's apply the very same medicine +--connection master +FLUSH BINARY LOGS; +--let $purge_to_binlog= query_get_value(SHOW MASTER STATUS, File, 1) +--eval PURGE BINARY LOGS TO '$purge_to_binlog'; +# with final removal of the extra domain +--eval FLUSH BINARY LOGS DELETE_DOMAIN_ID=($extra_domain_id) + +--connection slave +--source include/start_slave.inc + +# +# cleanup +# +--connection master +SET @@SESSION.gtid_domain_id=0; +DROP TABLE t; + +sync_slave_with_master; + +--source include/rpl_end.inc diff --git a/sql/lex.h b/sql/lex.h index 85bd20a5f37..6a1cb6653e9 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -179,6 +179,7 @@ static SYMBOL symbols[] = { { "DELAYED", SYM(DELAYED_SYM)}, { "DELAY_KEY_WRITE", SYM(DELAY_KEY_WRITE_SYM)}, { "DELETE", SYM(DELETE_SYM)}, + { "DELETE_DOMAIN_ID", SYM(DELETE_DOMAIN_ID_SYM)}, { "DESC", SYM(DESC)}, { "DESCRIBE", SYM(DESCRIBE)}, { "DES_KEY_FILE", SYM(DES_KEY_FILE)}, diff --git a/sql/log.cc b/sql/log.cc index a9f486d88c1..a0e5e908ca7 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -6622,6 +6622,120 @@ void MYSQL_BIN_LOG::checkpoint_and_purge(ulong binlog_id) purge(); } + +/** + Searches for the first (oldest) binlog file name in in the binlog index. + + @param[in,out] buf_arg pointer to a buffer to hold found + the first binary log file name + @return NULL on success, otherwise error message +*/ +static const char* get_first_binlog(char* buf_arg) +{ + IO_CACHE *index_file; + size_t length; + char fname[FN_REFLEN]; + const char* errmsg= NULL; + + DBUG_ENTER("get_first_binlog"); + + DBUG_ASSERT(mysql_bin_log.is_open()); + + mysql_bin_log.lock_index(); + + index_file=mysql_bin_log.get_index_file(); + if (reinit_io_cache(index_file, READ_CACHE, (my_off_t) 0, 0, 0)) + { + errmsg= "failed to create a cache on binlog index"; + goto end; + } + /* The file ends with EOF or empty line */ + if ((length=my_b_gets(index_file, fname, sizeof(fname))) <= 1) + { + errmsg= "empty binlog index"; + goto end; + } + else + { + fname[length-1]= 0; // Remove end \n + } + if (normalize_binlog_name(buf_arg, fname, false)) + { + errmsg= "cound not normalize the first file name in the binlog index"; + goto end; + } +end: + mysql_bin_log.unlock_index(); + + DBUG_RETURN(errmsg); +} + +/** + Check weather the gtid binlog state can safely remove gtid + domains passed as the argument. A safety condition is satisfied when + there are no events from the being deleted domains in the currently existing + binlog files. Upon successful check the supplied domains are removed + from @@gtid_binlog_state. The caller is supposed to rotate binlog so that + the active latest file won't have the deleted domains in its Gtid_list header. + + @param domain_drop_lex gtid domain id sequence from lex. + Passed as a pointer to dynamic array must be not empty + unless pointer value NULL. + @retval zero on success + @retval > 0 ineffective call none from the *non* empty + gtid domain sequence is deleted + @retval < 0 on error +*/ +static int do_delete_gtid_domain(DYNAMIC_ARRAY *domain_drop_lex) +{ + int rc= 0; + Gtid_list_log_event *glev= NULL; + char buf[FN_REFLEN]; + File file; + IO_CACHE cache; + const char* errmsg= NULL; + char errbuf[MYSQL_ERRMSG_SIZE]= {0}; + + if (!domain_drop_lex) + return 0; // still "effective" having empty domain sequence to delete + + DBUG_ASSERT(domain_drop_lex->elements > 0); + mysql_mutex_assert_owner(mysql_bin_log.get_log_lock()); + + if ((errmsg= get_first_binlog(buf)) != NULL) + goto end; + bzero((char*) &cache, sizeof(cache)); + if ((file= open_binlog(&cache, buf, &errmsg)) == (File) -1) + goto end; + errmsg= get_gtid_list_event(&cache, &glev); + end_io_cache(&cache); + mysql_file_close(file, MYF(MY_WME)); + + DBUG_EXECUTE_IF("inject_binlog_delete_domain_init_error", + errmsg= "injected error";); + if (errmsg) + goto end; + errmsg= rpl_global_gtid_binlog_state.drop_domain(domain_drop_lex, + glev, errbuf); + +end: + if (errmsg) + { + if (strlen(errmsg) > 0) + { + my_error(ER_BINLOG_CANT_DELETE_GTID_DOMAIN, MYF(0), errmsg); + rc= -1; + } + else + { + rc= 1; + } + } + delete glev; + + return rc; +} + /** The method is a shortcut of @c rotate() and @c purge(). LOCK_log is acquired prior to rotate and is released after it. @@ -6631,9 +6745,10 @@ void MYSQL_BIN_LOG::checkpoint_and_purge(ulong binlog_id) @retval nonzero - error in rotating routine. */ -int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate) +int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate, + DYNAMIC_ARRAY *domain_drop_lex) { - int error= 0; + int err_gtid=0, error= 0; ulong prev_binlog_id; DBUG_ENTER("MYSQL_BIN_LOG::rotate_and_purge"); bool check_purge= false; @@ -6641,7 +6756,14 @@ int MYSQL_BIN_LOG::rotate_and_purge(bool force_rotate) //todo: fix the macro def and restore safe_mutex_assert_not_owner(&LOCK_log); mysql_mutex_lock(&LOCK_log); prev_binlog_id= current_binlog_id; - if ((error= rotate(force_rotate, &check_purge))) + + if ((err_gtid= do_delete_gtid_domain(domain_drop_lex))) + { + // inffective attempt to delete merely skips rotate and purge + if (err_gtid < 0) + error= 1; // otherwise error is propagated the user + } + else if ((error= rotate(force_rotate, &check_purge))) check_purge= false; /* NOTE: Run purge_logs wo/ holding LOCK_log because it does not need @@ -10219,6 +10341,73 @@ TC_LOG_BINLOG::set_status_variables(THD *thd) } } + +/* + Find the Gtid_list_log_event at the start of a binlog. + + NULL for ok, non-NULL error message for error. + + If ok, then the event is returned in *out_gtid_list. This can be NULL if we + get back to binlogs written by old server version without GTID support. If + so, it means we have reached the point to start from, as no GTID events can + exist in earlier binlogs. +*/ +const char * +get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) +{ + Format_description_log_event init_fdle(BINLOG_VERSION); + Format_description_log_event *fdle; + Log_event *ev; + const char *errormsg = NULL; + + *out_gtid_list= NULL; + + if (!(ev= Log_event::read_log_event(cache, 0, &init_fdle, + opt_master_verify_checksum)) || + ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) + { + if (ev) + delete ev; + return "Could not read format description log event while looking for " + "GTID position in binlog"; + } + + fdle= static_cast<Format_description_log_event *>(ev); + + for (;;) + { + Log_event_type typ; + + ev= Log_event::read_log_event(cache, 0, fdle, opt_master_verify_checksum); + if (!ev) + { + errormsg= "Could not read GTID list event while looking for GTID " + "position in binlog"; + break; + } + typ= ev->get_type_code(); + if (typ == GTID_LIST_EVENT) + break; /* Done, found it */ + if (typ == START_ENCRYPTION_EVENT) + { + if (fdle->start_decryption((Start_encryption_log_event*) ev)) + errormsg= "Could not set up decryption for binlog."; + } + delete ev; + if (typ == ROTATE_EVENT || typ == STOP_EVENT || + typ == FORMAT_DESCRIPTION_EVENT || typ == START_ENCRYPTION_EVENT) + continue; /* Continue looking */ + + /* We did not find any Gtid_list_log_event, must be old binlog. */ + ev= NULL; + break; + } + + delete fdle; + *out_gtid_list= static_cast<Gtid_list_log_event *>(ev); + return errormsg; +} + struct st_mysql_storage_engine binlog_storage_engine= { MYSQL_HANDLERTON_INTERFACE_VERSION }; diff --git a/sql/log.h b/sql/log.h index bf076fae31d..3026ca11e31 100644 --- a/sql/log.h +++ b/sql/log.h @@ -755,7 +755,7 @@ public: int update_log_index(LOG_INFO* linfo, bool need_update_threads); int rotate(bool force_rotate, bool* check_purge); void checkpoint_and_purge(ulong binlog_id); - int rotate_and_purge(bool force_rotate); + int rotate_and_purge(bool force_rotate, DYNAMIC_ARRAY* drop_gtid_domain= NULL); /** Flush binlog cache and synchronize to disk. @@ -1165,4 +1165,9 @@ static inline TC_LOG *get_tc_log_implementation() return &tc_log_mmap; } + +class Gtid_list_log_event; +const char * +get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list); + #endif /* LOG_H */ diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 51df8f1a789..7b1acf17ef5 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -26,7 +26,7 @@ #include "key.h" #include "rpl_gtid.h" #include "rpl_rli.h" - +#include "log_event.h" const LEX_STRING rpl_gtid_slave_state_table_name= { C_STRING_WITH_LEN("gtid_slave_pos") }; @@ -1728,6 +1728,155 @@ end: return res; } +/** + Remove domains supplied by the first argument from binlog state. + Removal is done for any domain whose last gtids (from all its servers) match + ones in Gtid list event of the 2nd argument. + + @param ids gtid domain id sequence, may contain dups + @param glev pointer to Gtid list event describing + the match condition + @param errbuf [out] pointer to possible error message array + + @retval NULL as success when at least one domain is removed + @retval "" empty string to indicate ineffective call + when no domains removed + @retval NOT EMPTY string otherwise an error message +*/ +const char* +rpl_binlog_state::drop_domain(DYNAMIC_ARRAY *ids, + Gtid_list_log_event *glev, + char* errbuf) +{ + DYNAMIC_ARRAY domain_unique; // sequece (unsorted) of unique element*:s + rpl_binlog_state::element* domain_unique_buffer[16]; + ulong k, l; + const char* errmsg= NULL; + + DBUG_ENTER("rpl_binlog_state::drop_domain"); + + my_init_dynamic_array2(&domain_unique, + sizeof(element*), domain_unique_buffer, + sizeof(domain_unique_buffer) / sizeof(element*), 4, 0); + + mysql_mutex_lock(&LOCK_binlog_state); + + /* + Gtid list is supposed to come from a binlog's Gtid_list event and + therefore should be a subset of the current binlog state. That is + for every domain in the list the binlog state contains a gtid with + sequence number not less than that of the list. + Exceptions of this inclusion rule are: + A. the list may still refer to gtids from already deleted domains. + Files containing them must have been purged whereas the file + with the list is not yet. + B. out of order groups were injected + C. manually build list of binlog files violating the inclusion + constraint. + While A is a normal case (not necessarily distinguishable from C though), + B and C may require the user's attention so any (incl the A's suspected) + inconsistency is diagnosed and *warned*. + */ + for (l= 0, errbuf[0]= 0; l < glev->count; l++, errbuf[0]= 0) + { + rpl_gtid* rb_state_gtid= find_nolock(glev->list[l].domain_id, + glev->list[l].server_id); + if (!rb_state_gtid) + sprintf(errbuf, + "missing gtids from the '%u-%u' domain-server pair which is " + "referred to in the gtid list describing an earlier state. Ignore " + "if the domain ('%u') was already explicitly deleted", + glev->list[l].domain_id, glev->list[l].server_id, + glev->list[l].domain_id); + else if (rb_state_gtid->seq_no < glev->list[l].seq_no) + sprintf(errbuf, + "having a gtid '%u-%u-%llu' which is less than " + "the '%u-%u-%llu' of the gtid list describing an earlier state. " + "The state may have been affected by manually injecting " + "a lower sequence number gtid or via replication", + rb_state_gtid->domain_id, rb_state_gtid->server_id, + rb_state_gtid->seq_no, glev->list[l].domain_id, + glev->list[l].server_id, glev->list[l].seq_no); + if (strlen(errbuf)) // use strlen() as cheap flag + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_BINLOG_CANT_DELETE_GTID_DOMAIN, + "The current gtid binlog state is incompatible with " + "a former one %s.", errbuf); + } + + /* + For each domain_id from ids + when no such domain in binlog state + warn && continue + For each domain.server's last gtid + when not locate the last gtid in glev.list + error out binlog state can't change + otherwise continue + */ + for (ulong i= 0; i < ids->elements; i++) + { + rpl_binlog_state::element *elem= NULL; + ulong *ptr_domain_id; + bool not_match; + + ptr_domain_id= (ulong*) dynamic_array_ptr(ids, i); + elem= (rpl_binlog_state::element *) + my_hash_search(&hash, (const uchar *) ptr_domain_id, 0); + if (!elem) + { + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_BINLOG_CANT_DELETE_GTID_DOMAIN, + "The gtid domain being deleted ('%lu') is not in " + "the current binlog state", *ptr_domain_id); + continue; + } + + for (not_match= true, k= 0; k < elem->hash.records; k++) + { + rpl_gtid *d_gtid= (rpl_gtid *)my_hash_element(&elem->hash, k); + for (ulong l= 0; l < glev->count && not_match; l++) + not_match= !(*d_gtid == glev->list[l]); + } + + if (not_match) + { + sprintf(errbuf, "binlog files may contain gtids from the domain ('%lu') " + "being deleted. Make sure to first purge those files", + *ptr_domain_id); + errmsg= errbuf; + goto end; + } + // compose a sequence of unique pointers to domain object + for (k= 0; k < domain_unique.elements; k++) + { + if ((rpl_binlog_state::element*) dynamic_array_ptr(&domain_unique, k) + == elem) + break; // domain_id's elem has been already in + } + if (k == domain_unique.elements) // proven not to have duplicates + insert_dynamic(&domain_unique, (uchar*) &elem); + } + + // Domain removal from binlog state + for (k= 0; k < domain_unique.elements; k++) + { + rpl_binlog_state::element *elem= *(rpl_binlog_state::element**) + dynamic_array_ptr(&domain_unique, k); + my_hash_free(&elem->hash); + my_hash_delete(&hash, (uchar*) elem); + } + + DBUG_ASSERT(strlen(errbuf) == 0); + + if (domain_unique.elements == 0) + errmsg= ""; + +end: + mysql_mutex_unlock(&LOCK_binlog_state); + delete_dynamic(&domain_unique); + + DBUG_RETURN(errmsg); +} slave_connection_state::slave_connection_state() { diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index ece6effbef6..79d566bddbf 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -34,6 +34,13 @@ struct rpl_gtid uint64 seq_no; }; +inline bool operator==(const rpl_gtid& lhs, const rpl_gtid& rhs) +{ + return + lhs.domain_id == rhs.domain_id && + lhs.server_id == rhs.server_id && + lhs.seq_no == rhs.seq_no; +}; enum enum_gtid_skip_type { GTID_SKIP_NOT, GTID_SKIP_STANDALONE, GTID_SKIP_TRANSACTION @@ -93,6 +100,7 @@ struct gtid_waiting { class Relay_log_info; struct rpl_group_info; +class Gtid_list_log_event; /* Replication slave state. @@ -256,6 +264,7 @@ struct rpl_binlog_state rpl_gtid *find_nolock(uint32 domain_id, uint32 server_id); rpl_gtid *find(uint32 domain_id, uint32 server_id); rpl_gtid *find_most_recent(uint32 domain_id); + const char* drop_domain(DYNAMIC_ARRAY *ids, Gtid_list_log_event *glev, char*); }; diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index d335b0b420b..15ec98045ea 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7142,3 +7142,5 @@ ER_NO_EIS_FOR_FIELD ER_WARN_AGGFUNC_DEPENDENCE eng "Aggregate function '%-.192s)' of SELECT #%d belongs to SELECT #%d" ukr "Агрегатна функція '%-.192s)' з SELECTу #%d належить до SELECTу #%d" +ER_BINLOG_CANT_DELETE_GTID_DOMAIN + eng "Could not delete gtid domain. Reason: %s." diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 8edd9b3fbd8..63ab6b5d046 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -800,6 +800,7 @@ void lex_end_stage2(LEX *lex) /* Reset LEX_MASTER_INFO */ lex->mi.reset(lex->sql_command == SQLCOM_CHANGE_MASTER); + delete_dynamic(&lex->delete_gtid_domain); DBUG_VOID_RETURN; } @@ -2878,6 +2879,10 @@ LEX::LEX() INITIAL_LEX_PLUGIN_LIST_SIZE, 0); reset_query_tables_list(TRUE); mi.init(); + init_dynamic_array2(&delete_gtid_domain, sizeof(ulong*), + gtid_domain_static_buffer, + initial_gtid_domain_buffer_size, + initial_gtid_domain_buffer_size, 0); } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 116ac815ec0..b57fba08b47 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -2726,6 +2726,13 @@ public: */ Item *limit_rows_examined; ulonglong limit_rows_examined_cnt; + /** + Holds a set of domain_ids for deletion at FLUSH..DELETE_DOMAIN_ID + */ + DYNAMIC_ARRAY delete_gtid_domain; + static const ulong initial_gtid_domain_buffer_size= 16; + ulong gtid_domain_static_buffer[initial_gtid_domain_buffer_size]; + inline void set_limit_rows_examined() { if (limit_rows_examined) diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc index d68ce96dc85..73dd9679ed7 100644 --- a/sql/sql_reload.cc +++ b/sql/sql_reload.cc @@ -153,7 +153,10 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, tmp_write_to_binlog= 0; if (mysql_bin_log.is_open()) { - if (mysql_bin_log.rotate_and_purge(true)) + DYNAMIC_ARRAY *drop_gtid_domain= + (thd && (thd->lex->delete_gtid_domain.elements > 0)) ? + &thd->lex->delete_gtid_domain : NULL; + if (mysql_bin_log.rotate_and_purge(true, drop_gtid_domain)) *write_to_binlog= -1; if (WSREP_ON) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 764047e4720..b5cca334891 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -30,7 +30,7 @@ #include <my_dir.h> #include "rpl_handler.h" #include "debug_sync.h" - +#include "log.h" // get_gtid_list_event enum enum_gtid_until_state { GTID_UNTIL_NOT_DONE, @@ -875,72 +875,6 @@ get_binlog_list(MEM_ROOT *memroot) DBUG_RETURN(current_list); } -/* - Find the Gtid_list_log_event at the start of a binlog. - - NULL for ok, non-NULL error message for error. - - If ok, then the event is returned in *out_gtid_list. This can be NULL if we - get back to binlogs written by old server version without GTID support. If - so, it means we have reached the point to start from, as no GTID events can - exist in earlier binlogs. -*/ -static const char * -get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) -{ - Format_description_log_event init_fdle(BINLOG_VERSION); - Format_description_log_event *fdle; - Log_event *ev; - const char *errormsg = NULL; - - *out_gtid_list= NULL; - - if (!(ev= Log_event::read_log_event(cache, 0, &init_fdle, - opt_master_verify_checksum)) || - ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) - { - if (ev) - delete ev; - return "Could not read format description log event while looking for " - "GTID position in binlog"; - } - - fdle= static_cast<Format_description_log_event *>(ev); - - for (;;) - { - Log_event_type typ; - - ev= Log_event::read_log_event(cache, 0, fdle, opt_master_verify_checksum); - if (!ev) - { - errormsg= "Could not read GTID list event while looking for GTID " - "position in binlog"; - break; - } - typ= ev->get_type_code(); - if (typ == GTID_LIST_EVENT) - break; /* Done, found it */ - if (typ == START_ENCRYPTION_EVENT) - { - if (fdle->start_decryption((Start_encryption_log_event*) ev)) - errormsg= "Could not set up decryption for binlog."; - } - delete ev; - if (typ == ROTATE_EVENT || typ == STOP_EVENT || - typ == FORMAT_DESCRIPTION_EVENT || typ == START_ENCRYPTION_EVENT) - continue; /* Continue looking */ - - /* We did not find any Gtid_list_log_event, must be old binlog. */ - ev= NULL; - break; - } - - delete fdle; - *out_gtid_list= static_cast<Gtid_list_log_event *>(ev); - return errormsg; -} - /* Check if every GTID requested by the slave is contained in this (or a later) diff --git a/sql/sql_repl.h b/sql/sql_repl.h index e2000bbca73..37acff3141f 100644 --- a/sql/sql_repl.h +++ b/sql/sql_repl.h @@ -82,7 +82,6 @@ int rpl_append_gtid_state(String *dest, bool use_binlog); int rpl_load_gtid_state(slave_connection_state *state, bool use_binlog); bool rpl_gtid_pos_check(THD *thd, char *str, size_t len); bool rpl_gtid_pos_update(THD *thd, char *str, size_t len); - #else struct LOAD_FILE_IO_CACHE : public IO_CACHE { }; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 3fda8832f6b..f379364e453 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1182,6 +1182,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); %token DELAYED_SYM %token DELAY_KEY_WRITE_SYM %token DELETE_SYM /* SQL-2003-R */ +%token DELETE_DOMAIN_ID_SYM %token DESC /* SQL-2003-N */ %token DESCRIBE /* SQL-2003-R */ %token DES_KEY_FILE @@ -1951,6 +1952,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); parse_vcol_expr vcol_opt_specifier vcol_opt_attribute vcol_opt_attribute_list vcol_attribute explainable_command + opt_delete_gtid_domain END_OF_INPUT %type <NONE> call sp_proc_stmts sp_proc_stmts1 sp_proc_stmt @@ -12762,7 +12764,7 @@ flush_option: { Lex->type|= REFRESH_GENERAL_LOG; } | SLOW LOGS_SYM { Lex->type|= REFRESH_SLOW_LOG; } - | BINARY LOGS_SYM + | BINARY LOGS_SYM opt_delete_gtid_domain { Lex->type|= REFRESH_BINARY_LOG; } | RELAY LOGS_SYM optional_connection_name { @@ -12819,6 +12821,24 @@ opt_table_list: | table_list {} ; +opt_delete_gtid_domain: + /* empty */ {} + | DELETE_DOMAIN_ID_SYM '=' '(' delete_domain_id_list ')' + {} + ; +delete_domain_id_list: + /* Empty */ + | delete_domain_id + | delete_domain_id_list ',' delete_domain_id + ; + +delete_domain_id: + ulong_num + { + insert_dynamic(&Lex->delete_gtid_domain, (uchar*) &($1)); + } + ; + optional_flush_tables_arguments: /* empty */ {$$= 0;} | AND_SYM DISABLE_SYM CHECKPOINT_SYM {$$= REFRESH_CHECKPOINT; } |