--source include/have_innodb.inc --source include/have_debug.inc --source include/have_debug_sync.inc --source include/master-slave.inc --echo *** MDEV-7847: "Slave worker thread retried transaction 10 time(s) in vain, giving up", followed by replication hanging *** --echo *** MDEV-7882: Excessive transaction retry in parallel replication *** --connection server_1 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t7 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; CREATE TABLE t8 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; --save_master_pos --connection server_2 --sync_with_master --source include/stop_slave.inc SET @old_mode= @@GLOBAL.slave_parallel_mode; SET GLOBAL slave_parallel_mode='conservative'; SET @old_threads= @@GLOBAL.slave_parallel_threads; SET GLOBAL slave_parallel_threads=40; SET @old_transaction_retries= @@GLOBAL.slave_transaction_retries; SET GLOBAL slave_transaction_retries=5; # Using dbug error injection, we artificially create event groups with a lot of # conflicting transactions in each event group. The bugs were originally seen # "in the wild" with transactions that did not conflict on the master, and only # conflicted very rarely on the slave (maybe some edge case with InnoDB btree # page splits or something like that). The event groups here loosely reflect # the structure of the original failure's group commits. --connection server_1 INSERT INTO t7 VALUES (1,1), (2,2), (3,3), (4,4), (5,5); SET @old_dbug= @@SESSION.debug_dbug; SET @commit_id= 42; SET SESSION debug_dbug="+d,binlog_force_commit_id"; INSERT INTO t8 VALUES (1,1); INSERT INTO t8 VALUES (2,2); INSERT INTO t8 VALUES (3,3); INSERT INTO t8 VALUES (4,4); INSERT INTO t8 VALUES (5,5); INSERT INTO t8 VALUES (6,6); INSERT INTO t8 VALUES (7,7); INSERT INTO t8 VALUES (8,8); UPDATE t7 SET b=9 WHERE a=3; UPDATE t7 SET b=10 WHERE a=3; UPDATE t7 SET b=11 WHERE a=3; INSERT INTO t8 VALUES (12,12); INSERT INTO t8 VALUES (13,13); UPDATE t7 SET b=14 WHERE a=3; UPDATE t7 SET b=15 WHERE a=3; INSERT INTO t8 VALUES (16,16); UPDATE t7 SET b=17 WHERE a=3; INSERT INTO t8 VALUES (18,18); INSERT INTO t8 VALUES (19,19); UPDATE t7 SET b=20 WHERE a=3; INSERT INTO t8 VALUES (21,21); UPDATE t7 SET b=22 WHERE a=3; INSERT INTO t8 VALUES (23,24); INSERT INTO t8 VALUES (24,24); UPDATE t7 SET b=25 WHERE a=3; INSERT INTO t8 VALUES (26,26); UPDATE t7 SET b=27 WHERE a=3; BEGIN; INSERT INTO t8 VALUES (28,28); INSERT INTO t8 VALUES (29,28), (30,28); INSERT INTO t8 VALUES (31,28); INSERT INTO t8 VALUES (32,28); INSERT INTO t8 VALUES (33,28); INSERT INTO t8 VALUES (34,28); INSERT INTO t8 VALUES (35,28); INSERT INTO t8 VALUES (36,28); INSERT INTO t8 VALUES (37,28); INSERT INTO t8 VALUES (38,28); INSERT INTO t8 VALUES (39,28); INSERT INTO t8 VALUES (40,28); INSERT INTO t8 VALUES (41,28); INSERT INTO t8 VALUES (42,28); COMMIT; SET @commit_id=43; INSERT INTO t8 VALUES (43,43); INSERT INTO t8 VALUES (44,44); UPDATE t7 SET b=45 WHERE a=3; INSERT INTO t8 VALUES (46,46); INSERT INTO t8 VALUES (47,47); UPDATE t7 SET b=48 WHERE a=3; INSERT INTO t8 VALUES (49,49); INSERT INTO t8 VALUES (50,50); SET @commit_id=44; INSERT INTO t8 VALUES (51,51); INSERT INTO t8 VALUES (52,52); UPDATE t7 SET b=53 WHERE a=3; INSERT INTO t8 VALUES (54,54); INSERT INTO t8 VALUES (55,55); UPDATE t7 SET b=56 WHERE a=3; INSERT INTO t8 VALUES (57,57); UPDATE t7 SET b=58 WHERE a=3; INSERT INTO t8 VALUES (58,58); INSERT INTO t8 VALUES (59,59); INSERT INTO t8 VALUES (60,60); INSERT INTO t8 VALUES (61,61); UPDATE t7 SET b=62 WHERE a=3; INSERT INTO t8 VALUES (63,63); INSERT INTO t8 VALUES (64,64); INSERT INTO t8 VALUES (65,65); INSERT INTO t8 VALUES (66,66); UPDATE t7 SET b=67 WHERE a=3; INSERT INTO t8 VALUES (68,68); UPDATE t7 SET b=69 WHERE a=3; UPDATE t7 SET b=70 WHERE a=3; UPDATE t7 SET b=71 WHERE a=3; INSERT INTO t8 VALUES (72,72); UPDATE t7 SET b=73 WHERE a=3; UPDATE t7 SET b=74 WHERE a=3; UPDATE t7 SET b=75 WHERE a=3; UPDATE t7 SET b=76 WHERE a=3; INSERT INTO t8 VALUES (77,77); UPDATE t7 SET b=78 WHERE a=3; INSERT INTO t8 VALUES (79,79); UPDATE t7 SET b=80 WHERE a=3; INSERT INTO t8 VALUES (81,81); UPDATE t7 SET b=82 WHERE a=3; INSERT INTO t8 VALUES (83,83); UPDATE t7 SET b=84 WHERE a=3; SET @commit_id=45; INSERT INTO t8 VALUES (85,85); UPDATE t7 SET b=86 WHERE a=3; INSERT INTO t8 VALUES (87,87); SET @commit_id=46; INSERT INTO t8 VALUES (88,88); INSERT INTO t8 VALUES (89,89); INSERT INTO t8 VALUES (90,90); SET SESSION debug_dbug=@old_dbug; INSERT INTO t8 VALUES (91,91); INSERT INTO t8 VALUES (92,92); INSERT INTO t8 VALUES (93,93); INSERT INTO t8 VALUES (94,94); INSERT INTO t8 VALUES (95,95); INSERT INTO t8 VALUES (96,96); INSERT INTO t8 VALUES (97,97); INSERT INTO t8 VALUES (98,98); INSERT INTO t8 VALUES (99,99); SELECT * FROM t7 ORDER BY a; SELECT * FROM t8 ORDER BY a; --source include/save_master_gtid.inc --connection server_2 --source include/start_slave.inc --source include/sync_with_master_gtid.inc SELECT * FROM t7 ORDER BY a; SELECT * FROM t8 ORDER BY a; --echo *** MDEV-8302: Duplicate key with parallel replication *** --connection server_2 --source include/stop_slave.inc /* Inject a small sleep which makes the race easier to hit. */ SET @old_dbug=@@GLOBAL.debug_dbug; SET GLOBAL debug_dbug="+d,inject_mdev8302"; --connection server_1 INSERT INTO t7 VALUES (100,1), (101,2), (102,3), (103,4), (104,5); # Artificially create a bunch of group commits with conflicting transactions. # The bug happened when T1 and T2 was in one group commit, and T3 was in the # following group commit. T2 is a DELETE of a row with same primary key as a # row that T3 inserts. T1 and T2 can conflict, causing T2 to be deadlock # killed after starting to commit. The bug was that T2 could roll back before # doing unmark_start_commit(); this could allow T3 to run before the retry # of T2, causing duplicate key violation. SET @old_dbug= @@SESSION.debug_dbug; SET @commit_id= 20000; SET SESSION debug_dbug="+d,binlog_force_commit_id"; --let $n = 100 --disable_query_log while ($n) { eval UPDATE t7 SET b=b+1 WHERE a=100+($n MOD 5); eval DELETE FROM t7 WHERE a=100+($n MOD 5); SET @commit_id = @commit_id + 1; eval INSERT INTO t7 VALUES (100+($n MOD 5), $n); SET @commit_id = @commit_id + 1; dec $n; } --enable_query_log SET SESSION debug_dbug=@old_dbug; SELECT * FROM t7 ORDER BY a; --source include/save_master_gtid.inc --connection server_2 --source include/start_slave.inc --source include/sync_with_master_gtid.inc SELECT * FROM t7 ORDER BY a; --source include/stop_slave.inc SET GLOBAL debug_dbug=@old_dbug; SET GLOBAL slave_parallel_mode=@old_mode; SET GLOBAL slave_parallel_threads=@old_threads; SET GLOBAL slave_transaction_retries=@old_transaction_retries; # Clean up. --source include/start_slave.inc SET DEBUG_SYNC= 'RESET'; --connection server_1 DROP TABLE t7,t8; SET DEBUG_SYNC= 'RESET'; --source include/rpl_end.inc