diff options
author | Brandon Nesterenko <brandon.nesterenko@mariadb.com> | 2021-09-14 12:58:04 -0600 |
---|---|---|
committer | Brandon Nesterenko <brandon.nesterenko@mariadb.com> | 2021-11-01 16:10:54 -0600 |
commit | f591888fdfe2b4ea49bad6c5521cac33a154c786 (patch) | |
tree | 4898092394bef25509c206e1306900a00fb14510 | |
parent | 303ed1a153630138ebefe3079f509aeb3bcac264 (diff) | |
download | mariadb-git-f591888fdfe2b4ea49bad6c5521cac33a154c786.tar.gz |
MDEV-4989: Support for GTID in mysqlbinlog
Intermediary commit addressing the following
1) Repeat --start-position or --stop-position specified
on the command line
2) Added non-strict mode for omitting out-of-order
warnings while processing events
3) Updated ID delegation gtid event filter to use MariaDB
HASH type instead of manual hash implementation
4) Updated mariadb-binlog event processing to activate or
deactivate an event only when a Gtid_log_event is
encountered, rather than keeping track of each event
in the group
-rw-r--r-- | client/mysqlbinlog.cc | 94 | ||||
-rw-r--r-- | man/mysqlbinlog.1 | 17 | ||||
-rw-r--r-- | mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc | 29 | ||||
-rw-r--r-- | mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_warn_cases.inc | 133 | ||||
-rw-r--r-- | mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result | 134 | ||||
-rw-r--r-- | mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window-master.opt | 2 | ||||
-rw-r--r-- | mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test | 133 | ||||
-rw-r--r-- | sql/log_event.h | 2 | ||||
-rw-r--r-- | sql/log_event_client.cc | 26 | ||||
-rw-r--r-- | sql/rpl_gtid.cc | 388 | ||||
-rw-r--r-- | sql/rpl_gtid.h | 151 | ||||
-rw-r--r-- | sql/sql_repl.cc | 3 |
12 files changed, 724 insertions, 388 deletions
diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 46de0b9e1eb..a1e5d07b11d 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -151,9 +151,6 @@ static ulonglong start_position= BIN_LOG_HEADER_SIZE, #define stop_position_mot ((my_off_t)stop_position) static Domain_gtid_event_filter *domain_gtid_filter= NULL; -static rpl_gtid *start_gtids, *stop_gtids; -static uint32 n_start_gtid_ranges= 0; -static uint32 n_stop_gtid_ranges= 0; static char *start_datetime_str, *stop_datetime_str; static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX; @@ -1034,9 +1031,11 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, if (ev_type == GTID_LIST_EVENT && is_gtid_filtering_enabled()) { Gtid_list_log_event *glev= (Gtid_list_log_event *)ev; - for (uint i= 0; i < n_start_gtid_ranges; i++) + size_t n_start_gtid_ranges= domain_gtid_filter->get_num_start_gtids(); + rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); + for (size_t i= 0; i < n_start_gtid_ranges; i++) { - for (uint k= 0; k < glev->count; k++) + for (size_t k= 0; k < glev->count; k++) { if (start_gtids[i].domain_id == glev->list[k].domain_id) { @@ -1045,20 +1044,18 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, "past binlog files; some of event groups of the domain %u " "may be missed from the expected output; the domain's gtid " "state of the current binlog file is %u-%u-%llu", - start_gtids[i].domain_id, - start_gtids[i].server_id, - start_gtids[i].seq_no, + PARAM_GTID(start_gtids[i]), glev->list[k].domain_id, - glev->list[k].domain_id, - glev->list[k].server_id, - glev->list[k].seq_no); + PARAM_GTID(glev->list[k])); break; } } } - for (uint i= 0; i < n_stop_gtid_ranges; i++) + size_t n_stop_gtid_ranges= domain_gtid_filter->get_num_stop_gtids(); + rpl_gtid *stop_gtids= domain_gtid_filter->get_stop_gtids(); + for (size_t i= 0; i < n_stop_gtid_ranges; i++) { - for (uint k= 0; k < glev->count; k++) + for (size_t k= 0; k < glev->count; k++) { if (stop_gtids[i].domain_id == glev->list[k].domain_id) { @@ -1067,18 +1064,17 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, "past binlog files so no output may be provided for " "the domain %u; the domain's gtid " "state of the current binlog file is %u-%u-%llu", - stop_gtids[i].domain_id, - stop_gtids[i].server_id, - stop_gtids[i].seq_no, - glev->list[k].domain_id, + PARAM_GTID(stop_gtids[i]), glev->list[k].domain_id, - glev->list[k].server_id, - glev->list[k].seq_no); + PARAM_GTID(glev->list[k])); break; } } } + my_free(start_gtids); + my_free(stop_gtids); } + /* If the binlog output should be filtered using GTIDs, test the new event group to see if its events should be written or ignored. @@ -1087,13 +1083,21 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, { Gtid_log_event *gle= (Gtid_log_event*) ev; rpl_gtid gtid; - gtid.domain_id= gle->domain_id; - gtid.server_id= gle->server_id; - gtid.seq_no= gle->seq_no; - if (!domain_gtid_filter->exclude(>id)) + set_rpl_gtid(>id, gle->domain_id, gle->server_id, gle->seq_no); + + if (domain_gtid_filter->has_finished()) + { + retval= OK_STOP; + goto end; + } + else if (!domain_gtid_filter->exclude(>id)) + { ev->activate_current_event_group(); - ev->last_gtid_standalone= - (gle->flags2 & Gtid_log_event::FL_STANDALONE) ? true : false; + } + else + { + ev->deactivate_current_event_group(); + } } /* @@ -1923,8 +1927,6 @@ static void cleanup() my_free(stop_datetime_str); my_free(start_pos_str); my_free(stop_pos_str); - my_free(start_gtids); - my_free(stop_gtids); free_root(&glob_root, MYF(0)); delete domain_gtid_filter; @@ -2180,7 +2182,12 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi break; case OPT_STOP_POSITION: { - stop_gtids= gtid_parse_string_to_list(stop_pos_str, strlen(stop_pos_str), + /* Stop position was already specified, so reset it and use the new list */ + if (domain_gtid_filter && domain_gtid_filter->get_num_stop_gtids() > 0) + domain_gtid_filter->clear_stop_gtids(); + + uint32 n_stop_gtid_ranges= 0; + rpl_gtid *stop_gtids= gtid_parse_string_to_list(stop_pos_str, strlen(stop_pos_str), &n_stop_gtid_ranges); if (stop_gtids == NULL) { @@ -2211,8 +2218,12 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi { rpl_gtid *stop_gtid= &stop_gtids[gtid_idx]; if (domain_gtid_filter->add_stop_gtid(stop_gtid)) + { + my_free(stop_gtids); return 1; + } } + my_free(stop_gtids); } else { @@ -2222,7 +2233,12 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi } case 'j': { - start_gtids= gtid_parse_string_to_list( + /* Start position was already specified, so reset it and use the new list */ + if (domain_gtid_filter && domain_gtid_filter->get_num_start_gtids() > 0) + domain_gtid_filter->clear_start_gtids(); + + uint32 n_start_gtid_ranges= 0; + rpl_gtid *start_gtids= gtid_parse_string_to_list( start_pos_str, strlen(start_pos_str), &n_start_gtid_ranges); if (start_gtids == NULL) @@ -2254,8 +2270,12 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi { rpl_gtid *start_gtid= &start_gtids[gtid_idx]; if (domain_gtid_filter->add_start_gtid(start_gtid)) + { + my_free(start_gtids); return 1; + } } + my_free(start_gtids); } else { @@ -2299,6 +2319,7 @@ static int parse_args(int *argc, char*** argv) if (domain_gtid_filter) { Log_event::enable_event_group_filtering(); + domain_gtid_filter->set_gtid_strict_mode(opt_gtid_strict_mode); } return 0; } @@ -2481,7 +2502,7 @@ static Exit_status check_master_version() goto err; } - if (n_start_gtid_ranges > 0) + if (domain_gtid_filter && domain_gtid_filter->get_num_start_gtids() > 0) { char str_buf[256]; String query_str(str_buf, sizeof(str_buf), system_charset_info); @@ -2489,7 +2510,10 @@ static Exit_status check_master_version() query_str.append(STRING_WITH_LEN("SET @slave_connect_state='"), system_charset_info); - for (uint32 gtid_idx = 0; gtid_idx < n_start_gtid_ranges; gtid_idx++) + size_t n_start_gtids= domain_gtid_filter->get_num_start_gtids(); + rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); + + for (size_t gtid_idx = 0; gtid_idx < n_start_gtids; gtid_idx++) { char buf[256]; rpl_gtid *start_gtid= &start_gtids[gtid_idx]; @@ -2498,9 +2522,10 @@ static Exit_status check_master_version() start_gtid->domain_id, start_gtid->server_id, start_gtid->seq_no); query_str.append(buf, strlen(buf)); - if (gtid_idx < n_start_gtid_ranges - 1) + if (gtid_idx < n_start_gtids - 1) query_str.append(','); } + my_free(start_gtids); query_str.append(STRING_WITH_LEN("'"), system_charset_info); if (unlikely(mysql_real_query(mysql, query_str.ptr(), query_str.length()))) @@ -3443,13 +3468,12 @@ int main(int argc, char** argv) } } - if (domain_gtid_filter) - domain_gtid_filter->write_warnings(stderr); - if (tmpdir.list) free_tmpdir(&tmpdir); if (result_file && result_file != stdout) my_fclose(result_file, MYF(0)); + if (domain_gtid_filter) + domain_gtid_filter->write_warnings(stderr); cleanup(); /* We cannot free DBUG, it is used in global destructors after exit(). */ my_end(my_end_arg | MY_DONT_FREE_DBUG); diff --git a/man/mysqlbinlog.1 b/man/mysqlbinlog.1 index 86b7b33a333..17c81809ce2 100644 --- a/man/mysqlbinlog.1 +++ b/man/mysqlbinlog.1 @@ -1013,6 +1013,23 @@ This option is useful for point\-in\-time recovery\&. .sp -1 .IP \(bu 2.3 .\} +.\" mysqlbinlog: gtid-strict-mode +.\" gtid-strict-mode option: mysqlbinlog +\fB\-\-gtid\-strict\-mode +.sp +Process binlog according to gtid-strict-mode specification\&. The start, stop +positions are verified to satisfy start < stop comparison condition\&. Sequence +numbers of any gtid domain must comprise monotically growing sequence\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} .\" mysqlbinlog: stop-datetime option .\" stop-datetime option: mysqlbinlog \fB\-\-stop\-datetime=\fR\fB\fIdatetime\fR\fR diff --git a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc index 0b2e3180a62..a80507d6e42 100644 --- a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc +++ b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc @@ -284,4 +284,33 @@ if (0 < `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'te --let SEARCH_FILE=$log_error_ --let SEARCH_PATTERN=WARNING --source include/search_pattern_in_file.inc +DROP TABLE t1; + +--echo # Test Case 14: +--echo # Start position is specified twice on the command line +CREATE TABLE t1 (a int); +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | $MYSQL +if ($test2_t1_good_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (0 < `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} +DROP TABLE t1; + +--echo # Test Case 15: +--echo # Stop position is specified twice on the command line +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | $MYSQL +if ($test2_t1_good_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (0 < `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} DROP TABLE t1;
\ No newline at end of file diff --git a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_warn_cases.inc b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_warn_cases.inc new file mode 100644 index 00000000000..47b2874b03b --- /dev/null +++ b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_warn_cases.inc @@ -0,0 +1,133 @@ +# +# This file runs warning cases for providing GTIDs to --start-position and +# --stop-position arguments in mariadb-binlog +# +# param $is_strict_mode boolean (0 for false, 1 for true) to enable or +# disable strict mode for GTID processing +# + +--let MYSQLD_DATADIR=`select @@datadir` +--let OUT_FILE=$MYSQLTEST_VARDIR/tmp/binlog.out + +if ($is_strict_mode == 0) +{ + --let BINLOG_WARN_PARAM=--skip-gtid-strict-mode +} +if ($is_strict_mode == 1) +{ + --let BINLOG_WARN_PARAM=--gtid-strict-mode +} + +--let $log_error_ = $MYSQLTEST_VARDIR/tmp/out.err +--let SEARCH_FILE=$log_error_ + +--echo # +--echo # Warning Case 1: +--echo # A gap in sequence number at end of window results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (3); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 $BINLOG_WARN_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 $BINLOG_WARN_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=WARNING +--source include/search_pattern_in_file.inc +DROP TABLE t1; + +--echo # +--echo # Warning Case 2: +--echo # A gap in sequence number at beginning of window results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 $BINLOG_WARN_PARAM > log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 $BINLOG_WARN_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=WARNING +--source include/search_pattern_in_file.inc +DROP TABLE t1; + +--echo # +--echo # Warning Case 3: +--echo # A gap in sequence number within a window results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 $BINLOG_WARN_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 $BINLOG_WARN_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=WARNING +--source include/search_pattern_in_file.inc +DROP TABLE t1; + +--echo # +--echo # Warning Case 4: +--echo # When invoked with just --start-position, a gap in sequence number +--echo # after the provided GTID results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 $BINLOG_WARN_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 $BINLOG_WARN_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=WARNING +--source include/search_pattern_in_file.inc +DROP TABLE t1; + +--echo # +--echo # Warning Case 5: +--echo # When invoked with just --stop-position, a gap in sequence number +--echo # before the provided GTID results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 $BINLOG_WARN_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 $BINLOG_WARN_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=WARNING +--source include/search_pattern_in_file.inc +DROP TABLE t1; + +--echo # +--echo # Warning Case 6: +--echo # One domain has a sequence number gap but another does not results in +--echo # a warning +RESET MASTER; +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +INSERT INTO t2 values (2); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 $BINLOG_WARN_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 $BINLOG_WARN_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=WARNING +--source include/search_pattern_in_file.inc +DROP TABLE t1; +DROP TABLE t2;
\ No newline at end of file diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result index b1bf1e2a8b1..9938fbbc583 100644 --- a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result @@ -120,6 +120,15 @@ CREATE TABLE t1 (a int); # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --stop-position=0-1-2 | MYSQL NOT FOUND /WARNING/ in mysqld.1.err DROP TABLE t1; +# Test Case 14: +# Start position is specified twice on the command line +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# Test Case 15: +# Stop position is specified twice on the command line +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; ###################################### # Test Group 2 # Run test cases on remote host @@ -232,6 +241,15 @@ CREATE TABLE t1 (a int); # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --stop-position=0-1-2 | MYSQL NOT FOUND /WARNING/ in mysqld.1.err DROP TABLE t1; +# Test Case 14: +# Start position is specified twice on the command line +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# Test Case 15: +# Stop position is specified twice on the command line +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; ############################## # Error Cases ############################## @@ -253,9 +271,10 @@ DROP TABLE t1; # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2 --stop-position=0-1-1 # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --stop-position=0-1-1 # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 -############################## -# Warning Cases -############################## +#################################################### +# Warning Case Group 1 +# Run test cases with strict-gtid-mode +#################################################### # # Warning Case 1: # A gap in sequence number at end of window results in a warning @@ -268,7 +287,7 @@ INSERT INTO t1 values (2); SET @@session.gtid_seq_no= 5; INSERT INTO t1 values (3); FLUSH LOGS; -# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 --gtid-strict-mode 2> log_error_ > OUT_FILE FOUND 1 /WARNING/ in out.err DROP TABLE t1; # @@ -281,7 +300,7 @@ CREATE TABLE t1 (a int); SET @@session.gtid_seq_no= 3; INSERT INTO t1 values (1); FLUSH LOGS; -# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 > /home/brandon/workspace/server/build/mysql-test/var/tmp/out.err > MYSQLTEST_VARDIR/tmp/out.binlog +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --gtid-strict-mode > log_error_ > OUT_FILE FOUND 1 /WARNING/ in out.err DROP TABLE t1; # @@ -295,7 +314,7 @@ INSERT INTO t1 values (1); SET @@session.gtid_seq_no= 4; INSERT INTO t1 values (2); FLUSH LOGS; -# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 --gtid-strict-mode 2> log_error_ > OUT_FILE FOUND 1 /WARNING/ in out.err DROP TABLE t1; # @@ -310,7 +329,7 @@ INSERT INTO t1 values (1); SET @@session.gtid_seq_no= 4; INSERT INTO t1 values (2); FLUSH LOGS; -# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --gtid-strict-mode 2> log_error_ > OUT_FILE FOUND 1 /WARNING/ in out.err DROP TABLE t1; # @@ -325,7 +344,7 @@ INSERT INTO t1 values (1); SET @@session.gtid_seq_no= 4; INSERT INTO t1 values (2); FLUSH LOGS; -# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 --gtid-strict-mode 2> log_error_ > OUT_FILE FOUND 1 /WARNING/ in out.err DROP TABLE t1; # @@ -345,10 +364,107 @@ INSERT INTO t1 values (1); SET @@session.gtid_seq_no= 4; INSERT INTO t1 values (2); FLUSH LOGS; -# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 --gtid-strict-mode 2> log_error_ > OUT_FILE FOUND 1 /WARNING/ in out.err DROP TABLE t1; DROP TABLE t2; +#################################################### +# Warning Case Group 2 +# Run test cases with --skip-strict-gtid-mode +#################################################### +# +# Warning Case 1: +# A gap in sequence number at end of window results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +# +# Warning Case 2: +# A gap in sequence number at beginning of window results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --skip-gtid-strict-mode > log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +# +# Warning Case 3: +# A gap in sequence number within a window results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +# +# Warning Case 4: +# When invoked with just --start-position, a gap in sequence number +# after the provided GTID results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +# +# Warning Case 5: +# When invoked with just --stop-position, a gap in sequence number +# before the provided GTID results in a warning +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +# +# Warning Case 6: +# One domain has a sequence number gap but another does not results in +# a warning +RESET MASTER; +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +INSERT INTO t2 values (2); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +DROP TABLE t2; ############################## # Cleanup ############################## diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window-master.opt b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window-master.opt index d17999c07c1..7377dd80169 100644 --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window-master.opt +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window-master.opt @@ -1 +1 @@ ---timezone=GMT-8
\ No newline at end of file +--timezone=GMT-8 diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test index dedcfbf14aa..45f6944068c 100644 --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test @@ -34,6 +34,9 @@ # the --stop-position values are hit # Test Case 12) Scalar and GTID values can be used together for stop or start # position +# Test Case 13) Start position is delayed within the binlog +# Test Case 14) Start position is specified twice on the command line +# Test Case 15) Stop position is specified twice on the command line # # To validate for data consistency, each test case compares a checksum of # correct data against a variant created after replaying the binlog using @@ -57,6 +60,9 @@ # Warning Case 6) One domain has a sequence number gap but another does not # results in a warning # +# Note that warning cases are tested with both --strict-gtid-mode and +# --skip-gtid-strict-mode to ensure out of order warnings are hidden when not in +# strict mode. # # References: # MDEV-4989: Support for GTID in mysqlbinlog @@ -143,122 +149,19 @@ RESET MASTER; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 ---echo ############################## ---echo # Warning Cases ---echo ############################## ---let $log_error_ = $MYSQLTEST_VARDIR/tmp/out.err ---let SEARCH_FILE=$log_error_ - ---echo # ---echo # Warning Case 1: ---echo # A gap in sequence number at end of window results in a warning -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -INSERT INTO t1 values (1); -INSERT INTO t1 values (2); -SET @@session.gtid_seq_no= 5; -INSERT INTO t1 values (3); -FLUSH LOGS; ---echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 2> $log_error_ > $MYSQLTEST_VARDIR/tmp/out.binlog ---let SEARCH_PATTERN=WARNING ---source include/search_pattern_in_file.inc -DROP TABLE t1; +--echo #################################################### +--echo # Warning Case Group 1 +--echo # Run test cases with strict-gtid-mode +--echo #################################################### +--let is_strict_mode= 1 +--source include/mysqlbinlog_gtid_window_warn_cases.inc ---echo # ---echo # Warning Case 2: ---echo # A gap in sequence number at beginning of window results in a warning -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -SET @@session.gtid_seq_no= 3; -INSERT INTO t1 values (1); -FLUSH LOGS; ---echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 > $log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 2> $log_error_ > $MYSQLTEST_VARDIR/tmp/out.binlog ---let SEARCH_PATTERN=WARNING ---source include/search_pattern_in_file.inc -DROP TABLE t1; - ---echo # ---echo # Warning Case 3: ---echo # A gap in sequence number within a window results in a warning -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -INSERT INTO t1 values (1); -SET @@session.gtid_seq_no= 4; -INSERT INTO t1 values (2); -FLUSH LOGS; ---echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 --stop-position=0-1-4 2> $log_error_ > $MYSQLTEST_VARDIR/tmp/out.binlog ---let SEARCH_PATTERN=WARNING ---source include/search_pattern_in_file.inc -DROP TABLE t1; - ---echo # ---echo # Warning Case 4: ---echo # When invoked with just --start-position, a gap in sequence number ---echo # after the provided GTID results in a warning -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -INSERT INTO t1 values (1); -SET @@session.gtid_seq_no= 4; -INSERT INTO t1 values (2); -FLUSH LOGS; ---echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0 2> $log_error_ > $MYSQLTEST_VARDIR/tmp/out.binlog ---let SEARCH_PATTERN=WARNING ---source include/search_pattern_in_file.inc -DROP TABLE t1; - ---echo # ---echo # Warning Case 5: ---echo # When invoked with just --stop-position, a gap in sequence number ---echo # before the provided GTID results in a warning -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -INSERT INTO t1 values (1); -SET @@session.gtid_seq_no= 4; -INSERT INTO t1 values (2); -FLUSH LOGS; ---echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --stop-position=0-1-4 2> $log_error_ > $MYSQLTEST_VARDIR/tmp/out.binlog ---let SEARCH_PATTERN=WARNING ---source include/search_pattern_in_file.inc -DROP TABLE t1; - ---echo # ---echo # Warning Case 6: ---echo # One domain has a sequence number gap but another does not results in ---echo # a warning -RESET MASTER; -SET @@session.gtid_domain_id= 1; -SET @@session.server_id= 2; -CREATE TABLE t2 (a int); -INSERT INTO t2 values (1); -INSERT INTO t2 values (2); -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -INSERT INTO t1 values (1); -SET @@session.gtid_seq_no= 4; -INSERT INTO t1 values (2); -FLUSH LOGS; ---echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 2> log_error_ > MYSQLTEST_VARDIR/tmp/out.binlog ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-0,1-2-0 --stop-position=0-1-4,1-2-3 2> $log_error_ > $MYSQLTEST_VARDIR/tmp/out.binlog ---let SEARCH_PATTERN=WARNING ---source include/search_pattern_in_file.inc -DROP TABLE t1; -DROP TABLE t2; +--echo #################################################### +--echo # Warning Case Group 2 +--echo # Run test cases with --skip-strict-gtid-mode +--echo #################################################### +--let is_strict_mode= 0 +--source include/mysqlbinlog_gtid_window_warn_cases.inc --echo ############################## diff --git a/sql/log_event.h b/sql/log_event.h index abd9ddc89e7..520209d8fde 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -1311,8 +1311,6 @@ public: */ static my_bool m_is_event_group_filtering_enabled; - static my_bool last_gtid_standalone; // gtid type of last seen gtid - /* Notify that all events part of the current group should be printed */ diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index ba51631c447..1917a90a1e2 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -326,7 +326,6 @@ static inline bool is_enum_or_set_type(uint type) { my_bool Log_event::m_is_event_group_active= FALSE; my_bool Log_event::m_is_event_group_filtering_enabled= FALSE; -my_bool Log_event::last_gtid_standalone= FALSE; /* Log_event::print_header() @@ -2013,15 +2012,6 @@ bool Query_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) { Write_on_release_cache cache(&print_event_info->head_cache, file, 0, this); -#ifndef MYSQL_SERVER - /* - This marks the end of an event group. All events prior to this have been - printed, need to reset the tracking for future event groups - */ - if (is_commit() || is_rollback() || last_gtid_standalone) - deactivate_current_event_group(); -#endif - /** reduce the size of io cache so that the write function is called for every call to my_b_write(). @@ -2396,14 +2386,6 @@ bool Xid_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) Write_on_release_cache cache(&print_event_info->head_cache, file, Write_on_release_cache::FLUSH_F, this); -#ifndef MYSQL_SERVER - /* - This marks the end of an event group. All events prior to this have been - printed, need to reset the tracking for future event groups - */ - deactivate_current_event_group(); -#endif - if (!print_event_info->short_form) { char buf[64]; @@ -3941,14 +3923,6 @@ bool XA_prepare_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) Write_on_release_cache::FLUSH_F, this); m_xid.serialize(); -#ifndef MYSQL_SERVER - /* - All events prior to this have been printed, need to reset the tracking for - future event groups - */ - deactivate_current_event_group(); -#endif - if (!print_event_info->short_form) { print_header(&cache, print_event_info, FALSE); diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 56e6cacbc2a..27ee073795e 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -25,7 +25,6 @@ #include "sql_base.h" #include "sql_parse.h" #include "key.h" -#include "rpl_gtid.h" #include "rpl_rli.h" #include "slave.h" #include "log_event.h" @@ -1276,7 +1275,15 @@ rpl_slave_state::domain_to_gtid(uint32 domain_id, rpl_gtid *out_gtid) #endif -/* +void set_rpl_gtid(rpl_gtid *out, uint32 domain_id_arg, + uint32 server_id_arg, uint64 seq_no_arg) +{ + out->domain_id= domain_id_arg; + out->server_id= server_id_arg; + out->seq_no= seq_no_arg; +} + + /* Parse a GTID at the start of a string, and update the pointer to point at the first character after the parsed GTID. @@ -1305,9 +1312,7 @@ gtid_parser_helper(const char **ptr, const char *end, rpl_gtid *out_gtid) if (err != 0) return 1; - out_gtid->domain_id= (uint32) v1; - out_gtid->server_id= (uint32) v2; - out_gtid->seq_no= v3; + set_rpl_gtid(out_gtid, (uint32) v1, (uint32) v2, v3); *ptr= q; return 0; } @@ -3016,12 +3021,13 @@ gtid_waiting::remove_from_wait_queue(gtid_waiting::hash_element *he, #endif -Window_gtid_event_filter::Window_gtid_event_filter() : +Window_gtid_event_filter::Window_gtid_event_filter(my_bool *is_gtid_strict_mode_arg) : m_has_start(FALSE), m_has_stop(FALSE), m_is_active(FALSE), m_has_passed(FALSE), - m_warning_flags(0) + m_warning_flags(0), + m_is_gtid_strict_mode(is_gtid_strict_mode_arg) { // m_start and m_stop do not need initial values if unused } @@ -3042,8 +3048,7 @@ int Window_gtid_event_filter::set_start_gtid(rpl_gtid *start) { sql_print_error("stop position %d-%d-%llu is not strictly greater than " "its counterpart start position %d-%d-%llu", - m_stop.domain_id, m_stop.server_id, m_stop.seq_no, - start->domain_id, start->server_id, start->seq_no); + PARAM_GTID(m_stop), PARAM_GTID((*start))); err= 1; goto err; } @@ -3051,9 +3056,7 @@ int Window_gtid_event_filter::set_start_gtid(rpl_gtid *start) // Copy values m_has_start= TRUE; - m_start.domain_id= start->domain_id; - m_start.server_id= start->server_id; - m_start.seq_no= start->seq_no; + set_rpl_gtid(&m_start, PARAM_GTID((*start))); err: return err; @@ -3075,8 +3078,7 @@ int Window_gtid_event_filter::set_stop_gtid(rpl_gtid *stop) { sql_print_error("stop position %d-%d-%llu is not strictly greater than " "its counterpart start position %d-%d-%llu", - stop->domain_id, stop->server_id, stop->seq_no, - m_start.domain_id, m_start.server_id, m_start.seq_no); + PARAM_GTID((*stop)), PARAM_GTID(m_start)); err= 1; goto err; } @@ -3084,9 +3086,7 @@ int Window_gtid_event_filter::set_stop_gtid(rpl_gtid *stop) // Copy values m_has_stop= TRUE; - m_stop.domain_id= stop->domain_id; - m_stop.server_id= stop->server_id; - m_stop.seq_no= stop->seq_no; + set_rpl_gtid(&m_stop, PARAM_GTID((*stop))); err: return err; @@ -3109,11 +3109,7 @@ static inline my_bool is_gtid_at_or_before(rpl_gtid *boundary, void Window_gtid_event_filter::verify_gtid_is_expected(rpl_gtid *gtid) { if (gtid->seq_no != last_gtid_seen.seq_no + 1) - { m_warning_flags |= WARN_GTID_SEQUENCE_NUMBER_OUT_OF_ORDER; - } - - last_gtid_seen.seq_no++; } my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) @@ -3142,7 +3138,7 @@ my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) should_exclude= FALSE; last_gtid_seen= *gtid; } - else if (is_gtid_at_or_after(&m_start, gtid) && + else if ((m_has_start && is_gtid_at_or_after(&m_start, gtid)) && (!m_has_stop || is_gtid_at_or_before(&m_stop, gtid))) { m_is_active= TRUE; @@ -3153,9 +3149,8 @@ my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) last_gtid_seen= *gtid; DBUG_PRINT("gtid-event-filter", - ("Window: Begin (%d-%d-%llu, %d-%d-%llu]", m_start.domain_id, - m_start.server_id, m_start.seq_no, m_stop.domain_id, - m_stop.server_id, m_stop.seq_no)); + ("Window: Begin (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); /* As the start of the range is exclusive, if this gtid is the start of @@ -3165,6 +3160,14 @@ my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) should_exclude= TRUE; else should_exclude= FALSE; + + if (m_has_stop && gtid->seq_no == m_stop.seq_no) + { + m_has_passed= TRUE; + DBUG_PRINT("gtid-event-filter", + ("Window: End (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + } } } /* if (!m_is_active && !m_has_passed) */ else if (m_is_active && !m_has_passed) @@ -3174,15 +3177,16 @@ my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) in the results. Additionally check if we are at the end of the window. If no end of the window is provided, go indefinitely */ - verify_gtid_is_expected(gtid); + if (*m_is_gtid_strict_mode) + verify_gtid_is_expected(gtid); + should_exclude= FALSE; if (m_has_stop && is_gtid_at_or_after(&m_stop, gtid)) { DBUG_PRINT("gtid-event-filter", - ("Window: End (%d-%d-%llu, %d-%d-%llu]", m_start.domain_id, - m_start.server_id, m_start.seq_no, m_stop.domain_id, - m_stop.server_id, m_stop.seq_no)); + ("Window: End (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); m_is_active= FALSE; m_has_passed= TRUE; @@ -3195,6 +3199,8 @@ my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) should_exclude= TRUE; } } + + last_gtid_seen.seq_no= gtid->seq_no; } return should_exclude; @@ -3202,10 +3208,7 @@ my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) my_bool Window_gtid_event_filter::has_finished() { - if (!m_has_stop) - return FALSE; - - return last_gtid_seen.seq_no == m_stop.seq_no; + return m_has_stop ? m_has_passed : FALSE; } void Window_gtid_event_filter::write_warnings(FILE *out) @@ -3215,54 +3218,32 @@ void Window_gtid_event_filter::write_warnings(FILE *out) fprintf(out, "WARNING: " "Found out of order GTID sequence number during event processing\n"); } + + /* Reset after writing */ + m_warning_flags= 0; +} + +void free_gtid_filter_element(void *p) +{ + gtid_filter_element *gfe = (gtid_filter_element *) p; + if (gfe->filter) + delete gfe->filter; + my_free(gfe); } Id_delegating_gtid_event_filter::Id_delegating_gtid_event_filter() : m_num_explicit_filters(0), m_num_completed_filters(0) { - uint32 i; - - m_filter_id_mask= 0xf; - - m_filters_by_id= (gtid_filter_element **) my_malloc( - PSI_NOT_INSTRUMENTED, - (m_filter_id_mask + 1) * sizeof(gtid_filter_element *), - MYF(MY_WME) - ); - - if (m_filters_by_id == NULL) - my_error(ER_OUT_OF_RESOURCES, MYF(0)); - - for (i = 0; i <= m_filter_id_mask; i++) - { - m_filters_by_id[i]= NULL; - } + my_hash_init(PSI_INSTRUMENT_ME, &m_filters_by_id_hash, &my_charset_bin, 32, + offsetof(gtid_filter_element, identifier), sizeof(uint32), NULL, + free_gtid_filter_element, HASH_UNIQUE); m_default_filter= new Accept_all_gtid_filter(); } -/* - Deconstructor deletes: - 1) All Identifiable_gtid_event_filters added - 2) All gtid_filter_element allocations -*/ Id_delegating_gtid_event_filter::~Id_delegating_gtid_event_filter() { - uint32 i; - for (i = 0; i <= m_filter_id_mask; i++) - { - gtid_filter_element *filter_element= m_filters_by_id[i], - *filter_element_to_del= NULL; - while(filter_element) - { - filter_element_to_del= filter_element; - filter_element= filter_element->next; - delete filter_element_to_del->filter; - my_free(filter_element_to_del); - } - } - my_free(m_filters_by_id); - + my_hash_free(&m_filters_by_id_hash); delete m_default_filter; } @@ -3275,103 +3256,77 @@ void Id_delegating_gtid_event_filter::set_default_filter(Gtid_event_filter *filt } gtid_filter_element * -Id_delegating_gtid_event_filter::try_find_filter_element_for_id( - gtid_filter_identifier filter_id) -{ - // Add this into the domain id list - uint32 map_idx= filter_id & m_filter_id_mask; - gtid_filter_element *filter_idx= m_filters_by_id[map_idx]; - - /* Find list index to add this filter */ - while (filter_idx) - { - if (filter_idx->filter->get_filter_identifier() == filter_id) - break; - filter_idx= filter_idx->next; - } - - return filter_idx; -} - -gtid_filter_element * Id_delegating_gtid_event_filter::find_or_create_filter_element_for_id( gtid_filter_identifier filter_id) { - // Add this into the domain id list - uint32 map_idx= filter_id & m_filter_id_mask; - gtid_filter_element *filter_idx= m_filters_by_id[map_idx], - *prev_idx= NULL; - - /* Find list index to add this filter */ - while (filter_idx) - { - prev_idx= filter_idx; - if (filter_idx->filter->get_filter_identifier() == filter_id) - { - break; - } - prev_idx= filter_idx; - filter_idx= filter_idx->next; - } + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); - if (filter_idx == NULL) + if (!fe) { - // No other domain ids have filters that index here, create this one - filter_idx= (gtid_filter_element *) my_malloc( + gtid_filter_element *new_fe= (gtid_filter_element *) my_malloc( PSI_NOT_INSTRUMENTED, sizeof(gtid_filter_element), MYF(MY_WME)); - filter_idx->filter= NULL; - filter_idx->next= NULL; - - if (prev_idx == NULL) + new_fe->filter= NULL; + new_fe->next= NULL; + new_fe->identifier= filter_id; + if (my_hash_insert(&m_filters_by_id_hash, (uchar*) new_fe)) { - // This is the first filter in the bucket - m_filters_by_id[map_idx]= filter_idx; - } - else - { - // End of list, append filter list to tail - prev_idx->next= filter_idx; + my_free(new_fe); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; } + fe= new_fe; } - return filter_idx; + return fe; } my_bool Id_delegating_gtid_event_filter::has_finished() { + /* + If all user-defined filters have deactivated, we are effectively + deactivated + */ return m_num_completed_filters == m_num_explicit_filters; } -void Id_delegating_gtid_event_filter::write_warnings(FILE *out) +my_bool filter_write_warning(gtid_filter_element *fe, FILE *out) { - uint32 i; - for (i = 0; i <= m_filter_id_mask; i++) + if (fe && fe->filter) { - gtid_filter_element *filter_element= m_filters_by_id[i]; - while(filter_element && filter_element->filter) - { - filter_element->filter->write_warnings(out); - filter_element= filter_element->next; - } + fe->filter->write_warnings(out); } + + return FALSE; // no error +} + +void Id_delegating_gtid_event_filter::write_warnings(FILE *out) +{ + my_hash_iterate(&m_filters_by_id_hash, + (my_hash_walk_action) filter_write_warning, out); } my_bool Id_delegating_gtid_event_filter::exclude(rpl_gtid *gtid) { gtid_filter_identifier filter_id= get_id_from_gtid(gtid); - gtid_filter_element *filter_element= try_find_filter_element_for_id(filter_id); + gtid_filter_element *filter_element= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); Gtid_event_filter *filter= (filter_element ? filter_element->filter : m_default_filter); - my_bool ret= filter->exclude(gtid); + my_bool ret= TRUE; - /* - If this is an explicitly defined filter, e.g. Window-based filter, check - if it has completed, and update the counter accordingly if so. - */ - if (filter_element && filter->has_finished()) + if(!filter_element || !filter->has_finished()) { - m_num_completed_filters++; + ret= filter->exclude(gtid); + + /* + If this is an explicitly defined filter, e.g. Window-based filter, check + if it has completed, and update the counter accordingly if so. + */ + if (filter_element && filter->has_finished()) + m_num_completed_filters++; } + return ret; } @@ -3385,23 +3340,22 @@ Domain_gtid_event_filter::find_or_create_window_filter_for_id( if (filter_element->filter == NULL) { - // New filter - wgef= new Window_gtid_event_filter(); - filter_element->filter= new Identifiable_gtid_event_filter(domain_id, wgef); + /* New filter */ + wgef= new Window_gtid_event_filter(&m_is_gtid_strict_mode); + filter_element->filter= wgef; m_num_explicit_filters++; } else if (filter_element->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE) { - // We have an existing window filter here - wgef= (Window_gtid_event_filter *) - filter_element->filter->get_identified_filter(); + /* We have an existing window filter here */ + wgef= (Window_gtid_event_filter *) filter_element->filter; } else { - /* - We have an existing filter but it is not of window type so propogate NULL - filter - */ + /* + We have an existing filter but it is not of window type so propogate NULL + filter + */ sql_print_error("cannot subset domain id %d by position, another rule " "exists on that domain", domain_id); @@ -3417,9 +3371,18 @@ int Domain_gtid_event_filter::add_start_gtid(rpl_gtid *gtid) find_or_create_window_filter_for_id(gtid->domain_id); if (filter_to_update == NULL) + { err= 1; + } else - err= filter_to_update->set_start_gtid(gtid); + { + if (!(err= filter_to_update->set_start_gtid(gtid))) + { + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + insert_dynamic(&m_start_filters, (const void *) &fe); + } + } return err; } @@ -3431,9 +3394,134 @@ int Domain_gtid_event_filter::add_stop_gtid(rpl_gtid *gtid) find_or_create_window_filter_for_id(gtid->domain_id); if (filter_to_update == NULL) + { err= 1; + } else - err= filter_to_update->set_stop_gtid(gtid); + { + if (!(err= filter_to_update->set_stop_gtid(gtid))) + { + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + insert_dynamic(&m_stop_filters, (const void *) &fe); + } + } return err; -}
\ No newline at end of file +} + +rpl_gtid *Domain_gtid_event_filter::get_start_gtids() +{ + rpl_gtid *gtid_list; + uint32 i; + size_t n_start_gtids= get_num_start_gtids(); + + gtid_list= (rpl_gtid *) my_malloc( + PSI_INSTRUMENT_ME, n_start_gtids * sizeof(rpl_gtid), MYF(MY_WME)); + + for (i = 0; i < n_start_gtids; i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + rpl_gtid win_start_gtid= wgef->get_start_gtid(); + set_rpl_gtid(>id_list[i], PARAM_GTID(win_start_gtid)); + } + + return gtid_list; +} + +rpl_gtid *Domain_gtid_event_filter::get_stop_gtids() +{ + rpl_gtid *gtid_list; + uint32 i; + size_t n_stop_gtids= get_num_stop_gtids(); + + gtid_list= (rpl_gtid *) my_malloc( + PSI_INSTRUMENT_ME, n_stop_gtids * sizeof(rpl_gtid), MYF(MY_WME)); + + for (i = 0; i < n_stop_gtids; i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + rpl_gtid win_stop_gtid= wgef->get_stop_gtid(); + set_rpl_gtid(>id_list[i], PARAM_GTID(win_stop_gtid)); + } + + return gtid_list; +} + +void Domain_gtid_event_filter::clear_start_gtids() +{ + uint32 i; + + for (i = 0; i < get_num_start_gtids(); i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + if (wgef->has_stop()) + { + /* + Don't delete the whole filter if it already has a stop position attached + */ + wgef->clear_start_pos(); + } + else + { + /* + This domain only has a stop, so delete the whole filter + */ + my_hash_delete(&m_filters_by_id_hash, (uchar *) fe); + m_num_explicit_filters--; + } + } + + reset_dynamic(&m_start_filters); +} + +void Domain_gtid_event_filter::clear_stop_gtids() +{ + uint32 i; + + for (i = 0; i < get_num_stop_gtids(); i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + if (wgef->has_start()) + { + /* + Don't delete the whole filter if it already has a start position attached + */ + wgef->clear_stop_pos(); + } + else + { + /* + This domain only has a start, so delete the whole filter + */ + my_hash_delete(&m_filters_by_id_hash, (uchar *) fe); + m_num_explicit_filters--; + } + } + + reset_dynamic(&m_stop_filters); +} diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index 3e39e5f7823..86751d3c371 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -28,6 +28,7 @@ extern const LEX_CSTRING rpl_gtid_slave_state_table_name; class String; #define GTID_MAX_STR_LENGTH (10+1+10+1+20) +#define PARAM_GTID(G) G.domain_id, G.server_id, G.seq_no struct rpl_gtid { @@ -382,6 +383,8 @@ extern rpl_gtid *gtid_parse_string_to_list(const char *p, size_t len, uint32 *out_len); extern rpl_gtid *gtid_unpack_string_to_list(const char *p, size_t len, uint32 *out_len); +extern void set_rpl_gtid(rpl_gtid *out, uint32 domain_id, uint32 server_id, + uint64 seq_no); /* Interface to support different methods of filtering log events by GTID @@ -449,39 +452,6 @@ public: }; /* - A sub-class of Gtid_event_filter which allows for quick identification - of potentially applicable filters for arbitrary GTIDs. -*/ -typedef uint32 gtid_filter_identifier; -class Identifiable_gtid_event_filter : public Gtid_event_filter -{ - -public: - Identifiable_gtid_event_filter(gtid_filter_identifier filter_id, - Gtid_event_filter *filter) - : m_filter_id(filter_id), m_filter(filter){}; - - ~Identifiable_gtid_event_filter() { - delete m_filter; - }; - - Gtid_event_filter *get_identified_filter() { return m_filter; } - gtid_filter_identifier get_filter_identifier() { return m_filter_id; } - - /* - Inherited functionality uses composition to call the pass-through filter - */ - my_bool exclude(rpl_gtid *gtid) { return m_filter->exclude(gtid); } - uint32 get_filter_type() { return m_filter->get_filter_type(); } - my_bool has_finished() { return m_filter->has_finished(); } - void write_warnings(FILE *out) { return m_filter->write_warnings(out); } - -protected: - gtid_filter_identifier m_filter_id; - Gtid_event_filter *m_filter; -}; - -/* A filter implementation that passes through events between two GTIDs, m_start (exclusive) and m_stop (inclusive). @@ -500,7 +470,7 @@ protected: class Window_gtid_event_filter : public Gtid_event_filter { public: - Window_gtid_event_filter(); + Window_gtid_event_filter(my_bool *is_gtid_strict_mode); ~Window_gtid_event_filter() {} my_bool exclude(rpl_gtid*); @@ -523,6 +493,27 @@ public: uint32 get_filter_type() { return WINDOW_GTID_FILTER_TYPE; } + + /* + Getter/setter methods + */ + my_bool has_start() { return m_has_start; } + my_bool has_stop() { return m_has_stop; } + rpl_gtid get_start_gtid() { return m_start; } + rpl_gtid get_stop_gtid() { return m_stop; } + + void clear_start_pos() + { + m_has_start= FALSE; + set_rpl_gtid(&m_start, 0, 0, 0); + } + + void clear_stop_pos() + { + m_has_stop= FALSE; + set_rpl_gtid(&m_stop, 0, 0, 0); + } + protected: /* @@ -579,6 +570,13 @@ private: filtering */ uint32 m_warning_flags; + + /* + is_gtid_strict_mode: presents additional warnings in strict mode. This + points to some controller boolean which determines + whether or not gtid_strict_mode is enabled or not. + */ + my_bool *m_is_gtid_strict_mode; }; /* @@ -586,9 +584,11 @@ private: if two filters have identifiers that lead to the same hash, they will be put into a linked list. */ +typedef uint32 gtid_filter_identifier; typedef struct _gtid_filter_element { - Identifiable_gtid_event_filter *filter; + Gtid_event_filter *filter; + gtid_filter_identifier identifier; /* Used for HASH lookup */ struct _gtid_filter_element *next; } gtid_filter_element; @@ -622,19 +622,10 @@ protected: uint32 m_num_explicit_filters; uint32 m_num_completed_filters; - uint32 m_filter_id_mask; Gtid_event_filter *m_default_filter; - /* - To reduce time to find a gtid window, they are indexed by domain_id. More - specifically, domain_ids are arranged into m_filter_id_mask+1 buckets, and - each bucket is a linked list of gtid_filter_elements that share the same - index. The index itself is found by a bitwise and, i.e. - some_rpl_gtid.domain_id & m_filter_id_mask - */ - gtid_filter_element **m_filters_by_id; + HASH m_filters_by_id_hash; - gtid_filter_element *try_find_filter_element_for_id(gtid_filter_identifier); gtid_filter_element *find_or_create_filter_element_for_id(gtid_filter_identifier); }; @@ -643,14 +634,33 @@ protected: domain id of a GTID. Additional helper functions include: - add_start_gtid(GTID) : adds a start GTID position to this filter, to be - identified by its domain id - add_stop_gtid(GTID) : adds a stop GTID position to this filter, to be - identified by its domain id + add_start_gtid(GTID) : adds a start GTID position to this filter, to be + identified by its domain id + add_stop_gtid(GTID) : adds a stop GTID position to this filter, to be + identified by its domain id + clear_start_gtids() : removes existing GTID start positions + clear_stop_gtids() : removes existing GTID stop positions + get_start_gtids() : gets all added GTID start positions + get_stop_gtids() : gets all added GTID stop positions + get_num_start_gtids() : gets the count of added GTID start positions + get_num_stop_gtids() : gets the count of added GTID stop positions */ class Domain_gtid_event_filter : public Id_delegating_gtid_event_filter { public: + Domain_gtid_event_filter() + : m_is_gtid_strict_mode(0) + { + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_start_filters, + sizeof(gtid_filter_element), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_stop_filters, + sizeof(gtid_filter_element), 8, 8, MYF(0)); + } + ~Domain_gtid_event_filter() + { + delete_dynamic(&m_start_filters); + delete_dynamic(&m_stop_filters); + } /* Returns the domain id of from the input GTID @@ -674,7 +684,50 @@ public: */ int add_stop_gtid(rpl_gtid *gtid); + /* + If start or stop position is respecified, we remove all existing values + and start over with the new specification. + */ + void clear_start_gtids(); + void clear_stop_gtids(); + + /* + Return list of all GTIDs used as start position. + + Note that this list is allocated and it is up to the user to free it + */ + rpl_gtid *get_start_gtids(); + + /* + Return list of all GTIDs used as stop position. + + Note that this list is allocated and it is up to the user to free it + */ + rpl_gtid *get_stop_gtids(); + + size_t get_num_start_gtids() { return m_start_filters.elements; } + size_t get_num_stop_gtids() { return m_stop_filters.elements; } + + /* + Enable or disable gtid_strict_mode for GTID sequence number processing. + */ + void set_gtid_strict_mode(my_bool gtid_strict_mode_arg) + { + m_is_gtid_strict_mode= gtid_strict_mode_arg; + } + private: + DYNAMIC_ARRAY m_start_filters; + DYNAMIC_ARRAY m_stop_filters; + + /* + This controls whether gtid_strict_mode is enabled or disabled for all + child filters, e.g. of type Window_gtid_event_filter. More specifically, + they point to this variable, so when it changes, the behavior of all + children using this value changes. + */ + my_bool m_is_gtid_strict_mode; + Window_gtid_event_filter *find_or_create_window_filter_for_id(gtid_filter_identifier); }; diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index d884dad876c..90fdce1b56f 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2839,7 +2839,8 @@ static int send_one_binlog_file(binlog_send_info *info, return 1; } -void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, ushort flags) +void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, + ushort flags) { LOG_INFO linfo; |