diff options
author | Brandon Nesterenko <brandon.nesterenko@mariadb.com> | 2021-12-05 15:50:46 -0700 |
---|---|---|
committer | Brandon Nesterenko <brandon.nesterenko@mariadb.com> | 2021-12-05 16:15:44 -0700 |
commit | 78a9d49907ba8f23a0317d9aa09ef0a45d0609c6 (patch) | |
tree | 5b2aa951b753a823df91f1b1bea45a53a91c27b5 | |
parent | 52114b71a83b033282644cd373862c1335f32e5b (diff) | |
download | mariadb-git-bb-10.7-MDEV-4989.tar.gz |
MDEV-4989bb-10.7-MDEV-4989
Renamed Gtid_stream_validator to
Binlog_gtid_state_validator
Refactored --stop-position verification to be inside
Binlog_gtid_state_validator
If --start-position is provided as a file position,
we disabled GTID state validation
Changed GTID state verification between different
binary files to be enabled with -vvv
-rw-r--r-- | client/mysqlbinlog.cc | 115 | ||||
-rw-r--r-- | mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc | 107 | ||||
-rw-r--r-- | mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc | 8 | ||||
-rw-r--r-- | mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result | 243 | ||||
-rw-r--r-- | mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result | 48 | ||||
-rw-r--r-- | mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test | 6 | ||||
-rw-r--r-- | mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test | 49 | ||||
-rw-r--r-- | sql/log_event_client.cc | 2 | ||||
-rw-r--r-- | sql/rpl_gtid.cc | 77 | ||||
-rw-r--r-- | sql/rpl_gtid.h | 25 |
10 files changed, 490 insertions, 190 deletions
diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index f586d789c73..e517ec31ea0 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -150,7 +150,7 @@ static ulonglong start_position= BIN_LOG_HEADER_SIZE, #define start_position_mot ((my_off_t)start_position) #define stop_position_mot ((my_off_t)stop_position) -static Gtid_stream_auditor *gtid_stream_auditor= NULL; +static Binlog_gtid_state_validator *gtid_state_validator= NULL; static Domain_gtid_event_filter *domain_gtid_filter= NULL; static char *start_datetime_str, *stop_datetime_str; @@ -1029,8 +1029,11 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, provided by --start-position and --stop-position), and the true start of the specified binary logs. The first GLLE provides the initial state of the binary logs. + + If --start-position is provided as a file offset, we don't want to skip + initial GTID state verification */ - static my_bool was_first_glle_processed= FALSE; + static my_bool was_first_glle_processed= start_position > BIN_LOG_HEADER_SIZE; /* Bypass flashback settings to event */ ev->is_flashback= opt_flashback; @@ -1038,7 +1041,12 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, ev->need_flashback_review= opt_flashback_review; #endif - /* Run time estimation of the output window configuration. */ + /* + Run time estimation of the output window configuration. + + Do not validate GLLE information is start position is provided as a file + offset. + */ if (ev_type == GTID_LIST_EVENT && ev->when) { Gtid_list_log_event *glev= (Gtid_list_log_event *)ev; @@ -1047,66 +1055,40 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, If this is the first Gtid_list_log_event, initialize the state of the GTID stream auditor to be consistent with the binary logs provided */ - if (gtid_stream_auditor && !was_first_glle_processed && glev->count && - gtid_stream_auditor->initialize_gtid_state(stderr, glev->list, - glev->count)) - goto err; - - /* - Verify that we are able to process events from this binlog. For example, - if our current GTID state is behind the state of the GLLE in the new log, - a user may have accidentally left out a log file to process. - */ - my_bool glle_state_invalid= FALSE; - if (gtid_stream_auditor) + if (gtid_state_validator && !was_first_glle_processed && glev->count) { - for (size_t k= 0; k < glev->count; k++) - { - if (gtid_stream_auditor->verify_gtid_state(stderr, &(glev->list[k]))) - glle_state_invalid= TRUE; - } - } + if (gtid_state_validator->initialize_gtid_state(stderr, glev->list, + glev->count)) + goto err; - /* - We only need to check stop positions against the first GLLE. If we are - supposed to stop in a domain where the binary logs have already passed, - error. - */ - if (!was_first_glle_processed && domain_gtid_filter) - { - size_t n_stop_gtid_ranges= domain_gtid_filter->get_num_stop_gtids(); - rpl_gtid *stop_gtids= domain_gtid_filter->get_stop_gtids(); - for (size_t i= 0; i < n_stop_gtid_ranges; i++) + if (domain_gtid_filter && !domain_gtid_filter->get_num_start_gtids()) { - for (size_t k= 0; k < glev->count; k++) + /* + We need to validate the GTID list from --stop-position because we + couldn't prove it intrinsically (i.e. using stop > start) + */ + rpl_gtid *stop_gtids= domain_gtid_filter->get_stop_gtids(); + size_t n_stop_gtids= domain_gtid_filter->get_num_stop_gtids(); + if (gtid_state_validator->verify_stop_state(stderr, stop_gtids, + n_stop_gtids)) { - if (stop_gtids[i].domain_id == glev->list[k].domain_id) - { - /* - Seq_no of 0 has special meaning to ignore all events on a domain - */ - if (stop_gtids[i].seq_no && - stop_gtids[i].seq_no <= glev->list[k].seq_no) - { - error( - "--stop-position gtid %u-%u-%llu does not exist in the " - "specified binlog files. The gtid state of domain %u in the " - "specified binary logs is %u-%u-%llu", - PARAM_GTID(stop_gtids[i]), glev->list[k].domain_id, - PARAM_GTID(glev->list[k])); - glle_state_invalid= TRUE; - } - break; - } + my_free(stop_gtids); + goto err; } + my_free(stop_gtids); } - my_free(stop_gtids); } - was_first_glle_processed= TRUE; + /* + Verify that we are able to process events from this binlog. For example, + if our current GTID state is behind the state of the GLLE in the new log, + a user may have accidentally left out a log file to process. + */ + if (gtid_state_validator && verbose >= 3) + for (size_t k= 0; k < glev->count; k++) + gtid_state_validator->verify_gtid_state(stderr, &(glev->list[k])); - if (glle_state_invalid) - goto err; + was_first_glle_processed= TRUE; } if (ev_type == GTID_EVENT) @@ -1142,24 +1124,24 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, If we don't care about ensuring GTID validity, just delete the auditor object to disable it for future checks. */ - if (gtid_stream_auditor) + if (gtid_state_validator) { if (!(opt_gtid_strict_mode || verbose >= 3)) { - delete gtid_stream_auditor; + delete gtid_state_validator; /* Explicitly reset to NULL to simplify checks on if auditing is enabled i.e. if it is defined, assume we want to use it */ - gtid_stream_auditor= NULL; + gtid_state_validator= NULL; } else { - gtid_err= gtid_stream_auditor->record(&ev_gtid); + gtid_err= gtid_state_validator->record(&ev_gtid); if (gtid_err && opt_gtid_strict_mode) { - gtid_stream_auditor->report(stderr, opt_gtid_strict_mode); + gtid_state_validator->report(stderr, opt_gtid_strict_mode); goto err; } } @@ -1170,8 +1152,7 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, If the GTID is ignored, it shouldn't count towards offset (rec_count should not be incremented) */ - if (!print_event_info->is_event_group_active() && - ev_type != FORMAT_DESCRIPTION_EVENT) + if (!print_event_info->is_event_group_active()) goto end_skip_count; /* @@ -1998,8 +1979,8 @@ static void cleanup() free_root(&glob_root, MYF(0)); delete domain_gtid_filter; - if (gtid_stream_auditor) - delete gtid_stream_auditor; + if (gtid_state_validator) + delete gtid_state_validator; delete binlog_filter; delete glob_description_event; @@ -2393,7 +2374,7 @@ static int parse_args(int *argc, char*** argv) (i.e. --skip-gtid-strict-mode or -vvv is not given), it is deleted when we are certain the initial gtid state is set. */ - gtid_stream_auditor= new Gtid_stream_auditor(); + gtid_state_validator= new Binlog_gtid_state_validator(); if (domain_gtid_filter) { @@ -2413,7 +2394,7 @@ static int parse_args(int *argc, char*** argv) */ size_t n_start_gtids= domain_gtid_filter->get_num_start_gtids(); rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); - gtid_stream_auditor->initialize_start_gtids(start_gtids, n_start_gtids); + gtid_state_validator->initialize_start_gtids(start_gtids, n_start_gtids); my_free(start_gtids); } @@ -3579,8 +3560,8 @@ int main(int argc, char** argv) are processed because it immediately errors (i.e. retval will be ERROR_STOP) */ - if (retval != ERROR_STOP && gtid_stream_auditor && - gtid_stream_auditor->report(stderr, opt_gtid_strict_mode)) + if (retval != ERROR_STOP && gtid_state_validator && + gtid_state_validator->report(stderr, opt_gtid_strict_mode)) retval= ERROR_STOP; cleanup(); diff --git a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc index e40a058648b..4f0b04333ab 100644 --- a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc +++ b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc @@ -317,4 +317,111 @@ FLUSH LOGS; DROP TABLE t1; RESET MASTER; +--echo # +--echo # Test Case 12: +--echo # Specifying multiple binary logs with a log-position start should +--echo # skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +--let $b2_start_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +--let BINLOG_FILE1= query_get_value(SHOW BINARY LOGS, Log_name, 1) +# Skip file 2 as input +--let BINLOG_FILE2= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE1 $MYSQLD_DATADIR/$BINLOG_FILE2 --start-position=$b2_start_pos $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +DROP TABLE t1; +DROP TABLE t2; + + +--echo # +--echo # Test Case 13: +--echo # If multiple binary logs should be specified but a middle log is +--echo # missing, we should detect that and warn when using -vvv + +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +--let $b2_start_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; + +--echo # +--echo # GLLE from each log for state reference +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--source include/show_gtid_list.inc +--let $binlog_file= +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source include/show_gtid_list.inc +--let $binlog_file= +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 3) +--source include/show_gtid_list.inc +--let $binlog_file= + +--let BINLOG_FILE1= query_get_value(SHOW BINARY LOGS, Log_name, 1) +# Skip file 2 as input +--let BINLOG_FILE3= query_get_value(SHOW BINARY LOGS, Log_name, 3) +--echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE1 $MYSQLD_DATADIR/$BINLOG_FILE3 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--echo # We should have two warnings about missing data from domains 0 and 1 if +--echo # -vvv is specified +--let SEARCH_FILE=$log_error_ +--let SEARCH_PATTERN=WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; + +--echo # +--echo # Test Case 14: +--echo # If a --stop-position GTID occurs before the first specified binlog's +--echo # GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; + +--echo # +--echo # GLLE from each log for state reference +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--source include/show_gtid_list.inc +--let $binlog_file= +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source include/show_gtid_list.inc +--let $binlog_file= +--echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 $BINLOG_STRICT_MODE_PARAM --stop-position=0-1-2 2> log_error_ > OUT_FILE +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE2 $BINLOG_STRICT_MODE_PARAM --stop-position=0-1-2 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=ERROR: --stop-position GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; + --remove_file $OUT_FILE diff --git a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc index f13c19d2812..b5cf849bc23 100644 --- a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc +++ b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc @@ -12,12 +12,12 @@ ## Initialize test data # +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; RESET MASTER; SET @@session.gtid_domain_id= 0; SET @@session.server_id= 1; --let $empty_binlog_pos= query_get_value(SHOW MASTER STATUS,Position, 1) -set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); -SET timestamp=@a; CREATE TABLE t1 (a int); SET timestamp=@a+1; INSERT INTO t1 values (1), (2); @@ -369,8 +369,8 @@ DROP TABLE t1; # --offset specification CREATE TABLE t1 (a int); ---echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=4 | MYSQL ---exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=4 | $MYSQL +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | $MYSQL if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) { die $data_inconsistent_err; diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result index 53e45b43a4b..4d42c88d6fe 100644 --- a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result @@ -227,6 +227,87 @@ FLUSH LOGS; FOUND 1 /ERROR: Found out of order GTID/ in out.err DROP TABLE t1; RESET MASTER; +# +# Test Case 12: +# Specifying multiple binary logs with a log-position start should +# skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos --gtid-strict-mode 2> log_error_ > OUT_FILE +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If multiple binary logs should be specified but a middle log is +# missing, we should detect that and warn when using -vvv +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-1] +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 --gtid-strict-mode 2> log_error_ > OUT_FILE +# We should have two warnings about missing data from domains 0 and 1 if +# -vvv is specified +NOT FOUND /WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in out.err +NOT FOUND /WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in out.err +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Test Case 14: +# If a --stop-position GTID occurs before the first specified binlog's +# GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-3] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 --gtid-strict-mode --stop-position=0-1-2 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: --stop-position GTID/ in out.err +DROP TABLE t1; #################################################### # Test Case Group 2 # @@ -454,6 +535,87 @@ FLUSH LOGS; FOUND 1 /WARNING: Found out of order GTID/ in out.err DROP TABLE t1; RESET MASTER; +# +# Test Case 12: +# Specifying multiple binary logs with a log-position start should +# skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If multiple binary logs should be specified but a middle log is +# missing, we should detect that and warn when using -vvv +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-1] +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +# We should have two warnings about missing data from domains 0 and 1 if +# -vvv is specified +FOUND 1 /WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in out.err +FOUND 1 /WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in out.err +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Test Case 14: +# If a --stop-position GTID occurs before the first specified binlog's +# GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-3] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 --skip-gtid-strict-mode -vvv --stop-position=0-1-2 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: --stop-position GTID/ in out.err +DROP TABLE t1; #################################################### # Test Case Group 3 # @@ -680,6 +842,87 @@ FLUSH LOGS; NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err DROP TABLE t1; RESET MASTER; +# +# Test Case 12: +# Specifying multiple binary logs with a log-position start should +# skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If multiple binary logs should be specified but a middle log is +# missing, we should detect that and warn when using -vvv +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-1] +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +# We should have two warnings about missing data from domains 0 and 1 if +# -vvv is specified +NOT FOUND /WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in out.err +NOT FOUND /WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in out.err +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Test Case 14: +# If a --stop-position GTID occurs before the first specified binlog's +# GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-3] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 --skip-gtid-strict-mode --stop-position=0-1-2 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: --stop-position GTID/ in out.err +DROP TABLE t1; ############################## # Cleanup ############################## diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result index dfff35ff061..dbf06f52a8b 100644 --- a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result @@ -6,11 +6,11 @@ RESET MASTER; # Test Group 1 # Run test cases on local log file ###################################### +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; RESET MASTER; SET @@session.gtid_domain_id= 0; SET @@session.server_id= 1; -set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); -SET timestamp=@a; CREATE TABLE t1 (a int); SET timestamp=@a+1; INSERT INTO t1 values (1), (2); @@ -153,7 +153,7 @@ DROP TABLE t1; # Start position with --offset=<n> skips n events after the first # GTID is found CREATE TABLE t1 (a int); -# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=4 | MYSQL +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | MYSQL DROP TABLE t1; # # Test Case 17: @@ -199,11 +199,11 @@ FOUND 2 /ERROR: Binary logs never reached expected GTID state/ in err.out # Test Group 2 # Run test cases on remote host ###################################### +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; RESET MASTER; SET @@session.gtid_domain_id= 0; SET @@session.server_id= 1; -set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); -SET timestamp=@a; CREATE TABLE t1 (a int); SET timestamp=@a+1; INSERT INTO t1 values (1), (2); @@ -346,7 +346,7 @@ DROP TABLE t1; # Start position with --offset=<n> skips n events after the first # GTID is found CREATE TABLE t1 (a int); -# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=4 | MYSQL +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | MYSQL DROP TABLE t1; # # Test Case 17: @@ -430,42 +430,6 @@ RESET MASTER; # Error Case 5: # User provides GTID ranges with repeated domain ids # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1,0-1-8 --stop-position=0-1-4,0-1-12 -# -# Error Case 6: -# If multiple binary logs should be specified but a middle log is -# missing, we should detect that and error -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -FLUSH LOGS; -INSERT INTO t1 values (1); -SET @@session.gtid_domain_id= 1; -SET @@session.server_id= 2; -CREATE TABLE t2 (a int); -FLUSH LOGS; -SET @@session.gtid_domain_id= 2; -SET @@session.server_id= 3; -CREATE TABLE t3 (a int); -FLUSH LOGS; -# GLLE from each log for state reference -show binlog events in 'master-bin.000001' limit 1,1; -Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000001 # Gtid_list 1 # [] -show binlog events in 'master-bin.000002' limit 1,1; -Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000002 # Gtid_list 1 # [0-1-1] -show binlog events in 'master-bin.000003' limit 1,1; -Log_name Pos Event_type Server_id End_log_pos Info -master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] -# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 2> err_out_ > tmp_out_ -# We should have two errors about missing data from domains 0 and 1 -FOUND 1 /ERROR: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in err.out -FOUND 1 /ERROR: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in err.out -DROP TABLE t1; -DROP TABLE t2; -DROP TABLE t3; -RESET MASTER; ############################## # Cleanup ############################## diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test index a202ce87a09..3036acff7af 100644 --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test @@ -28,6 +28,12 @@ # specification # Test Case 11) Strict mode warnings should be independent of # --start-timestamp option specification +# Test Case 12) Specifying multiple binary logs with a log-position start +# should skip GTID state verification +# Test Case 13) If multiple binary logs should be specified but a middle log +# is missing, we should detect that and warn when using -vvv +# Test Case 14) If a --stop-position GTID occurs before the first specified +# binlog's GLLE, error # # Note that test cases are tested under three scenarios: # 1) --gtid-strict-mode should error and immediately quit with error on out of diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test index 0cba088044c..b00d6e7b596 100644 --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test @@ -135,57 +135,8 @@ RESET MASTER; --exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1,0-1-8 --stop-position=0-1-4,0-1-12 ---echo # ---echo # Error Case 6: ---echo # If multiple binary logs should be specified but a middle log is ---echo # missing, we should detect that and error -RESET MASTER; -SET @@session.gtid_domain_id= 0; -SET @@session.server_id= 1; -CREATE TABLE t1 (a int); -FLUSH LOGS; -INSERT INTO t1 values (1); -SET @@session.gtid_domain_id= 1; -SET @@session.server_id= 2; -CREATE TABLE t2 (a int); -FLUSH LOGS; -SET @@session.gtid_domain_id= 2; -SET @@session.server_id= 3; -CREATE TABLE t3 (a int); -FLUSH LOGS; - ---echo # GLLE from each log for state reference ---let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 1) ---source include/show_gtid_list.inc ---let $binlog_file= ---let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 2) ---source include/show_gtid_list.inc ---let $binlog_file= ---let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 3) ---source include/show_gtid_list.inc ---let $binlog_file= - ---let BINLOG_FILE1= query_get_value(SHOW BINARY LOGS, Log_name, 1) -# Skip file 2 as input ---let BINLOG_FILE3= query_get_value(SHOW BINARY LOGS, Log_name, 3) ---echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 2> err_out_ > tmp_out_ ---error 1 ---exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE1 $MYSQLD_DATADIR/$BINLOG_FILE3 2> $err_out_ > $tmp_out_ ---echo # We should have two errors about missing data from domains 0 and 1 ---let SEARCH_FILE=$err_out_ ---let SEARCH_PATTERN=ERROR: Binary logs are missing data for domain 0[^\n]+the last seen event was ---source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=ERROR: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events ---source include/search_pattern_in_file.inc -DROP TABLE t1; -DROP TABLE t2; -DROP TABLE t3; -RESET MASTER; - - --echo ############################## --echo # Cleanup --echo ############################## --eval SET @@global.gtid_domain_id= $orig_gtid_domain_id --eval SET @@global.server_id= $orig_server_id ---remove_file $err_out_ diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index ef062edd281..bcae3f277fd 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -3790,7 +3790,7 @@ st_print_event_info::st_print_event_info() printed_fd_event=FALSE; file= 0; base64_output_mode=BASE64_OUTPUT_UNSPEC; - m_is_event_group_active= FALSE; + m_is_event_group_active= TRUE; m_is_event_group_filtering_enabled= FALSE; open_cached_file(&head_cache, NULL, NULL, 0, flags); open_cached_file(&body_cache, NULL, NULL, 0, flags); diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 8a7e995e02f..04f0eedaf63 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -2980,27 +2980,27 @@ gtid_waiting::remove_from_wait_queue(gtid_waiting::hash_element *he, void free_domain_lookup_element(void *p) { - struct Gtid_stream_auditor::audit_elem *audit_elem= - (struct Gtid_stream_auditor::audit_elem *) p; + struct Binlog_gtid_state_validator::audit_elem *audit_elem= + (struct Binlog_gtid_state_validator::audit_elem *) p; delete_dynamic(&audit_elem->late_gtids_previous); delete_dynamic(&audit_elem->late_gtids_real); my_free(audit_elem); } -Gtid_stream_auditor::Gtid_stream_auditor() +Binlog_gtid_state_validator::Binlog_gtid_state_validator() { my_hash_init(PSI_INSTRUMENT_ME, &m_audit_elem_domain_lookup, &my_charset_bin, 32, offsetof(struct audit_elem, domain_id), sizeof(uint32), NULL, free_domain_lookup_element, HASH_UNIQUE); } -Gtid_stream_auditor::~Gtid_stream_auditor() +Binlog_gtid_state_validator::~Binlog_gtid_state_validator() { my_hash_free(&m_audit_elem_domain_lookup); } -void Gtid_stream_auditor::initialize_start_gtids(rpl_gtid *start_gtids, - size_t n_gtids) +void Binlog_gtid_state_validator::initialize_start_gtids(rpl_gtid *start_gtids, + size_t n_gtids) { size_t i; for(i= 0; i < n_gtids; i++) @@ -3053,8 +3053,9 @@ void Gtid_stream_auditor::initialize_start_gtids(rpl_gtid *start_gtids, } } -my_bool Gtid_stream_auditor::initialize_gtid_state(FILE *out, rpl_gtid *gtids, - size_t n_gtids) +my_bool Binlog_gtid_state_validator::initialize_gtid_state(FILE *out, + rpl_gtid *gtids, + size_t n_gtids) { size_t i; my_bool err= FALSE; @@ -3076,7 +3077,7 @@ my_bool Gtid_stream_auditor::initialize_gtid_state(FILE *out, rpl_gtid *gtids, if (!audit_elem) { - Gtid_stream_auditor::error( + Binlog_gtid_state_validator::error( out, "Starting GTID position list does not specify an initial value " "for domain %u, whose events may be present in the requested binlog " @@ -3088,7 +3089,7 @@ my_bool Gtid_stream_auditor::initialize_gtid_state(FILE *out, rpl_gtid *gtids, if (audit_elem->start_gtid.seq_no < domain_state_gtid->seq_no) { - Gtid_stream_auditor::error( + Binlog_gtid_state_validator::error( out, "Binary logs are missing data for domain %u. Expected data to " "start from state %u-%u-%llu; however, the initial GTID state of " @@ -3105,7 +3106,42 @@ my_bool Gtid_stream_auditor::initialize_gtid_state(FILE *out, rpl_gtid *gtids, return err; } -my_bool Gtid_stream_auditor::verify_gtid_state(FILE *out, +my_bool Binlog_gtid_state_validator::verify_stop_state(FILE *out, + rpl_gtid *stop_gtids, + size_t n_stop_gtids) +{ + size_t i; + for(i= 0; i < n_stop_gtids; i++) + { + rpl_gtid *stop_gtid= &stop_gtids[i]; + + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(stop_gtid->domain_id), 0); + + /* + It is okay if stop gtid doesn't exist in current state because it will be treated + as a new domain + */ + if (audit_elem && stop_gtid->seq_no <= audit_elem->start_gtid.seq_no) + { + Binlog_gtid_state_validator::error( + out, + "--stop-position GTID %u-%u-%llu does not exist in the " + "specified binlog files. The current GTID state of domain %u in the " + "specified binary logs is %u-%u-%llu", + PARAM_GTID((*stop_gtid)), stop_gtid->domain_id, + PARAM_GTID(audit_elem->start_gtid)); + return TRUE; + } + } + + /* No issues with any GTIDs */ + return FALSE; +} + +my_bool +Binlog_gtid_state_validator::verify_gtid_state(FILE *out, rpl_gtid *domain_state_gtid) { struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( @@ -3114,12 +3150,13 @@ my_bool Gtid_stream_auditor::verify_gtid_state(FILE *out, if (!audit_elem) { - Gtid_stream_auditor::error( + Binlog_gtid_state_validator::warn( out, "Binary logs are missing data for domain %u. The current binary log " "specified its " "current state for this domain as %u-%u-%llu, but neither the " - "starting GTID position list nor any processed events have mentioned " + "starting GTID position list nor any processed events have " + "mentioned " "this domain.", domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid))); return TRUE; @@ -3127,7 +3164,7 @@ my_bool Gtid_stream_auditor::verify_gtid_state(FILE *out, if (audit_elem->last_gtid.seq_no < domain_state_gtid->seq_no) { - Gtid_stream_auditor::error( + Binlog_gtid_state_validator::warn( out, "Binary logs are missing data for domain %u. The current binary log " "state is %u-%u-%llu, but the last seen event was %u-%u-%llu.", @@ -3139,7 +3176,7 @@ my_bool Gtid_stream_auditor::verify_gtid_state(FILE *out, return FALSE; } -my_bool Gtid_stream_auditor::record(rpl_gtid *gtid) +my_bool Binlog_gtid_state_validator::record(rpl_gtid *gtid) { struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( &m_audit_elem_domain_lookup, (const uchar *) &(gtid->domain_id), 0); @@ -3210,8 +3247,8 @@ struct gtid_report_ctx static my_bool report_audit_findings(void *entry, void *report_ctx_arg) { - struct Gtid_stream_auditor::audit_elem *audit_el= - (struct Gtid_stream_auditor::audit_elem *) entry; + struct Binlog_gtid_state_validator::audit_elem *audit_el= + (struct Binlog_gtid_state_validator::audit_elem *) entry; struct gtid_report_ctx *report_ctx= (struct gtid_report_ctx *) report_ctx_arg; @@ -3221,9 +3258,9 @@ static my_bool report_audit_findings(void *entry, void *report_ctx_arg) void (*report_f)(FILE*, const char*, ...); if (is_strict_mode) - report_f= Gtid_stream_auditor::error; + report_f= Binlog_gtid_state_validator::error; else - report_f= Gtid_stream_auditor::warn; + report_f= Binlog_gtid_state_validator::warn; if (audit_el) { @@ -3254,7 +3291,7 @@ static my_bool report_audit_findings(void *entry, void *report_ctx_arg) return FALSE; } -my_bool Gtid_stream_auditor::report(FILE *out, my_bool is_strict_mode) +my_bool Binlog_gtid_state_validator::report(FILE *out, my_bool is_strict_mode) { struct gtid_report_ctx report_ctx; report_ctx.out_file= out; diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index 8b8de6b950a..a6a65ee1d4c 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -390,11 +390,16 @@ extern rpl_gtid *gtid_unpack_string_to_list(const char *p, size_t len, /* - Class which audits a stream of GTIDs (from arbitrary domains) to see if the - sequence numbers are out of order in any domains. This class silently - collects problem GTIDs and writes them as warnings in `report()`. + This class ensures that the GTID state of an event stream is consistent with + the set of provided binary log files. In particular, it has two concerns: + + 1) Ensuring that GTID events are monotonically increasing within each + domain + 2) Ensuring that the GTID state of the specified binary logs is consistent + both with the initial state that a user provides, and between + binary logs (if multiple are specified) */ -class Gtid_stream_auditor +class Binlog_gtid_state_validator { public: @@ -425,12 +430,12 @@ public: DYNAMIC_ARRAY late_gtids_previous; }; - Gtid_stream_auditor(); - ~Gtid_stream_auditor(); + Binlog_gtid_state_validator(); + ~Binlog_gtid_state_validator(); /* Initialize where we should start monitoring for invalid GTID entries - in the data stream. Note that these start positions must occur at or after + in the event stream. Note that these start positions must occur at or after a given binary logs GTID state (from Gtid_list_log_event) */ void initialize_start_gtids(rpl_gtid *start_gtids, size_t n_gtids); @@ -444,6 +449,12 @@ public: my_bool initialize_gtid_state(FILE *out, rpl_gtid *gtids, size_t n_gtids); /* + Ensures that the expected stop GTID positions exist within the specified + binary logs. + */ + my_bool verify_stop_state(FILE *out, rpl_gtid *stop_gtids, size_t n_stop_gtids); + + /* Ensure a GTID state (e.g., from a Gtid_list_log_event) is consistent with the current state of our auditing. For example, if we see a GTID from a Gtid_list_log_event that is ahead of our current state for that domain, we |