MDEV-28423: Galera IST is failing on Joiner node

This commit fixes an issue with IST handling in
version 10.9 which is a regression after MDEV-26971
and related to trying to get a non-existent "total"
tag on the IST branch (this tag is only defined in
SST mode).

6 files changed, 799 insertions, 27 deletions

diff --git a/mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff b/mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff
new file mode 100644
index 00000000000..8c84321e774
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff
@@ -0,0 +1,190 @@
+--- r/galera_ist_MDEV-28423.result
++++ r/galera_ist_MDEV-28423,debug.reject
+@@ -517,3 +517,187 @@
+ 1
+ DROP TABLE t1;
+ COMMIT;
++Performing State Transfer on a server that has been killed and restarted
++while a DDL was in progress on it
++connection node_1;
++CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 VALUES (1,'node1_committed_before');
++INSERT INTO t1 VALUES (2,'node1_committed_before');
++INSERT INTO t1 VALUES (3,'node1_committed_before');
++INSERT INTO t1 VALUES (4,'node1_committed_before');
++INSERT INTO t1 VALUES (5,'node1_committed_before');
++connection node_2;
++START TRANSACTION;
++INSERT INTO t1 VALUES (6,'node2_committed_before');
++INSERT INTO t1 VALUES (7,'node2_committed_before');
++INSERT INTO t1 VALUES (8,'node2_committed_before');
++INSERT INTO t1 VALUES (9,'node2_committed_before');
++INSERT INTO t1 VALUES (10,'node2_committed_before');
++COMMIT;
++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
++connection node_1;
++ALTER TABLE t1 ADD COLUMN f2 INTEGER;
++connection node_2;
++SET wsrep_sync_wait = 0;
++Killing server ...
++connection node_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during');
++COMMIT;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after');
++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after');
++connection node_2;
++Performing --wsrep-recover ...
++connection node_2;
++Starting server ...
++Using --wsrep-start-position when starting mysqld ...
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after');
++COMMIT;
++connection node_1;
++INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after');
++COMMIT;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after');
++COMMIT;
++connection node_1a_galera_st_kill_slave_ddl;
++INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after');
++ROLLBACK;
++SET AUTOCOMMIT=ON;
++SET SESSION wsrep_sync_wait=15;
++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++EXPECT_3
++3
++SELECT COUNT(*) AS EXPECT_35 FROM t1;
++EXPECT_35
++35
++SELECT * FROM t1;
++id	f1	f2
++1	node1_committed_before	NULL
++2	node1_committed_before	NULL
++3	node1_committed_before	NULL
++4	node1_committed_before	NULL
++5	node1_committed_before	NULL
++6	node2_committed_before	NULL
++7	node2_committed_before	NULL
++8	node2_committed_before	NULL
++9	node2_committed_before	NULL
++10	node2_committed_before	NULL
++11	node1_committed_during	NULL
++12	node1_committed_during	NULL
++13	node1_committed_during	NULL
++14	node1_committed_during	NULL
++15	node1_committed_during	NULL
++16	node1_to_be_committed_after	NULL
++17	node1_to_be_committed_after	NULL
++18	node1_to_be_committed_after	NULL
++19	node1_to_be_committed_after	NULL
++20	node1_to_be_committed_after	NULL
++26	node2_committed_after	NULL
++27	node2_committed_after	NULL
++28	node2_committed_after	NULL
++29	node2_committed_after	NULL
++30	node2_committed_after	NULL
++31	node1_to_be_committed_after	NULL
++32	node1_to_be_committed_after	NULL
++33	node1_to_be_committed_after	NULL
++34	node1_to_be_committed_after	NULL
++35	node1_to_be_committed_after	NULL
++36	node1_committed_after	NULL
++37	node1_committed_after	NULL
++38	node1_committed_after	NULL
++39	node1_committed_after	NULL
++40	node1_committed_after	NULL
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++COMMIT;
++connection node_1;
++SET AUTOCOMMIT=ON;
++SET SESSION wsrep_sync_wait=15;
++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++EXPECT_3
++3
++SELECT COUNT(*) AS EXPECT_35 FROM t1;
++EXPECT_35
++35
++SELECT * FROM t1;
++id	f1	f2
++1	node1_committed_before	NULL
++2	node1_committed_before	NULL
++3	node1_committed_before	NULL
++4	node1_committed_before	NULL
++5	node1_committed_before	NULL
++6	node2_committed_before	NULL
++7	node2_committed_before	NULL
++8	node2_committed_before	NULL
++9	node2_committed_before	NULL
++10	node2_committed_before	NULL
++11	node1_committed_during	NULL
++12	node1_committed_during	NULL
++13	node1_committed_during	NULL
++14	node1_committed_during	NULL
++15	node1_committed_during	NULL
++16	node1_to_be_committed_after	NULL
++17	node1_to_be_committed_after	NULL
++18	node1_to_be_committed_after	NULL
++19	node1_to_be_committed_after	NULL
++20	node1_to_be_committed_after	NULL
++26	node2_committed_after	NULL
++27	node2_committed_after	NULL
++28	node2_committed_after	NULL
++29	node2_committed_after	NULL
++30	node2_committed_after	NULL
++31	node1_to_be_committed_after	NULL
++32	node1_to_be_committed_after	NULL
++33	node1_to_be_committed_after	NULL
++34	node1_to_be_committed_after	NULL
++35	node1_to_be_committed_after	NULL
++36	node1_committed_after	NULL
++37	node1_committed_after	NULL
++38	node1_committed_after	NULL
++39	node1_committed_after	NULL
++40	node1_committed_after	NULL
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++DROP TABLE t1;
++COMMIT;
++SET GLOBAL debug_dbug = $debug_orig;
diff --git a/mysql-test/suite/galera/r/galera_ist_MDEV-28423.result b/mysql-test/suite/galera/r/galera_ist_MDEV-28423.result
new file mode 100644
index 00000000000..5a71b490a80
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_ist_MDEV-28423.result
@@ -0,0 +1,519 @@
+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+Performing State Transfer on a server that has been temporarily disconnected
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Unloading wsrep provider ...
+SET GLOBAL wsrep_cluster_address = '';
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_disconnect_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Loading wsrep provider ...
+disconnect node_2;
+connect node_2, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_disconnect_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that has been shut down cleanly and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Shutting down server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_shutdown_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (44,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_15 FROM t1;
+EXPECT_15
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_15 FROM t1;
+EXPECT_15
+35
+SELECT * from t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+Performing State Transfer on a server that has been killed and restarted
+connection node_1;
+CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (1,'node1_committed_before');
+INSERT INTO t1 VALUES (2,'node1_committed_before');
+INSERT INTO t1 VALUES (3,'node1_committed_before');
+INSERT INTO t1 VALUES (4,'node1_committed_before');
+INSERT INTO t1 VALUES (5,'node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (6,'node2_committed_before');
+INSERT INTO t1 VALUES (7,'node2_committed_before');
+INSERT INTO t1 VALUES (8,'node2_committed_before');
+INSERT INTO t1 VALUES (9,'node2_committed_before');
+INSERT INTO t1 VALUES (10,'node2_committed_before');
+COMMIT;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (11,'node1_committed_during');
+INSERT INTO t1 VALUES (12,'node1_committed_during');
+INSERT INTO t1 VALUES (13,'node1_committed_during');
+INSERT INTO t1 VALUES (14,'node1_committed_during');
+INSERT INTO t1 VALUES (15,'node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES (16,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (17,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (18,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (19,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (20,'node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (21,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (22,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (23,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (24,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (25,'node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (26,'node2_committed_after');
+INSERT INTO t1 VALUES (27,'node2_committed_after');
+INSERT INTO t1 VALUES (28,'node2_committed_after');
+INSERT INTO t1 VALUES (29,'node2_committed_after');
+INSERT INTO t1 VALUES (30,'node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES (31,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (32,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (33,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (34,'node1_to_be_committed_after');
+INSERT INTO t1 VALUES (35,'node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES (36,'node1_committed_after');
+INSERT INTO t1 VALUES (37,'node1_committed_after');
+INSERT INTO t1 VALUES (38,'node1_committed_after');
+INSERT INTO t1 VALUES (39,'node1_committed_after');
+INSERT INTO t1 VALUES (40,'node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave;
+INSERT INTO t1 VALUES (41,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (42,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (43,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (45,'node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES (46,'node1_to_be_rollbacked_after');
+ROLLBACK;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+connection node_1;
+SET AUTOCOMMIT=ON;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) AS EXPECT_35 FROM t1;
+EXPECT_35
+35
+SELECT * FROM t1;
+id	f1
+1	node1_committed_before
+2	node1_committed_before
+3	node1_committed_before
+4	node1_committed_before
+5	node1_committed_before
+6	node2_committed_before
+7	node2_committed_before
+8	node2_committed_before
+9	node2_committed_before
+10	node2_committed_before
+11	node1_committed_during
+12	node1_committed_during
+13	node1_committed_during
+14	node1_committed_during
+15	node1_committed_during
+16	node1_to_be_committed_after
+17	node1_to_be_committed_after
+18	node1_to_be_committed_after
+19	node1_to_be_committed_after
+20	node1_to_be_committed_after
+26	node2_committed_after
+27	node2_committed_after
+28	node2_committed_after
+29	node2_committed_after
+30	node2_committed_after
+31	node1_to_be_committed_after
+32	node1_to_be_committed_after
+33	node1_to_be_committed_after
+34	node1_to_be_committed_after
+35	node1_to_be_committed_after
+36	node1_committed_after
+37	node1_committed_after
+38	node1_committed_after
+39	node1_committed_after
+40	node1_committed_after
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
diff --git a/mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf
new file mode 100644
index 00000000000..691e52208b1
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf
@@ -0,0 +1,44 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+# server-id=101
+#wsrep-debug=1
+innodb_file_per_table
+innodb_autoinc_lock_mode=2
+#wsrep_sst_method=rsync
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:
+binlog_format=ROW
+core-file
+log-output=none
+wsrep_slave_threads=2
+wsrep_on=1
+gtid_strict_mode=1
+log_slave_updates=ON
+log_bin=binlog
+
+[mysqld.2]
+# server-id=102
+#wsrep-debug=1
+innodb_file_per_table
+innodb_autoinc_lock_mode=2
+#wsrep_sst_method=rsync
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:
+binlog_format=ROW
+core-file
+log-output=none
+wsrep_slave_threads=2
+wsrep_on=1
+gtid_strict_mode=1
+log_slave_updates=ON
+log_bin=binlog
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
diff --git a/mysql-test/suite/galera/t/galera_ist_MDEV-28423.test b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.test
new file mode 100644
index 00000000000..8668c4ce158
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_ist_MDEV-28423.test
@@ -0,0 +1,18 @@
+# MDEV-28423: Galera IST is failing on Joiner node
+
+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--source suite/galera/include/galera_st_disconnect_slave.inc
+--source suite/galera/include/galera_st_shutdown_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+--source include/auto_increment_offset_restore.inc
diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 87ed8392475..314d2349bcc 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -86,6 +86,7 @@ encrypt_threads=""
 encrypt_chunk=""
 
 readonly SECRET_TAG='secret'
+readonly TOTAL_TAG='total'
 
 # Required for backup locks
 # For backup locks it is 1 sent by joiner
@@ -324,7 +325,8 @@ get_transfer()
             if [ -z "$ssl_dhparams" ]; then
                 # Determine the socat version
                 SOCAT_VERSION=$(socat -V 2>&1 | \
-                                grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1)
+                                grep -m1 -owE '[0-9]+(\.[0-9]+)+' | \
+                                head -n1 || :)
                 if [ -z "$SOCAT_VERSION" ]; then
                     wsrep_log_error "******** FATAL ERROR ******************"
                     wsrep_log_error "* Cannot determine the socat version. *"
@@ -770,25 +772,26 @@ recv_joiner()
 
     if [ $checkf -eq 1 ]; then
         if [ ! -r "$MAGIC_FILE" ]; then
-            # this message should cause joiner to abort
+            # this message should cause joiner to abort:
             wsrep_log_error "receiving process ended without creating" \
-                            "'$MAGIC_FILE'"
-            wsrep_log_info "Contents of datadir"
+                            "magic file ($MAGIC_FILE)"
+            wsrep_log_info "Contents of datadir:"
             wsrep_log_info $(ls -l "$dir/"*)
             exit 32
         fi
 
         # check donor supplied secret
-        SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
-                 cut -d ' ' -f2)
+        SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :)
+        SECRET=$(trim_string "${SECRET#$SECRET_TAG}")
         if [ "$SECRET" != "$MY_SECRET" ]; then
             wsrep_log_error "Donor does not know my secret!"
             wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
             exit 32
         fi
 
-        # remove secret from the magic file
-        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+        # remove secret and total from the magic file
+        grep -v -E "^($SECRET_TAG|$TOTAL_TAG)[[:space:]]" -- \
+             "$MAGIC_FILE" > "$MAGIC_FILE.new"
         mv "$MAGIC_FILE.new" "$MAGIC_FILE"
     fi
 }
@@ -1256,8 +1259,7 @@ then
 
     recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
 
-    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
-    then
+    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
         wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \
                         "terminated unexpectedly."
         exit 32
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index 29e2b390e27..27d4e875674 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -804,8 +804,7 @@ EOF
         sleep 1
     done
 
-    if ! ps -p $MYSQLD_PID >/dev/null 2>&1
-    then
+    if ! ps -p $MYSQLD_PID >/dev/null 2>&1; then
         wsrep_log_error \
             "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly."
         kill -- -$MYSQLD_PID
@@ -813,24 +812,24 @@ EOF
         exit 32
     fi
 
-    if [ -r "$MAGIC_FILE" ]; then
-        if [ -n "$MY_SECRET" ]; then
-            # Check donor supplied secret:
-            SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
-                     cut -d ' ' -f2)
-            if [ "$SECRET" != "$MY_SECRET" ]; then
-                wsrep_log_error "Donor does not know my secret!"
-                wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
-                exit 32
-            fi
-        fi
-    else
+    if [ ! -r "$MAGIC_FILE" ]; then
         # This message should cause joiner to abort:
-        wsrep_log_info "rsync process ended without creating magic file"
-        echo "rsync process ended without creating '$MAGIC_FILE'"
+        wsrep_log_info "rsync process ended without creating" \
+                       "magic file ($MAGIC_FILE)"
         exit 32
     fi
 
+    if [ -n "$MY_SECRET" ]; then
+        # Check donor supplied secret:
+        SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :)
+        SECRET=$(trim_string "${SECRET#$SECRET_TAG}")
+        if [ "$SECRET" != "$MY_SECRET" ]; then
+            wsrep_log_error "Donor does not know my secret!"
+            wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
+            exit 32
+        fi
+    fi
+
     if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
         binlog_tar_present=0
         [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1
@@ -907,7 +906,7 @@ EOF
     if [ -n "$MY_SECRET" ]; then
         # remove secret from the magic file, and output
         # the UUID:seqno & wsrep_gtid_domain_id:
-        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE"
+        grep -v -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE"
     else
         # Output the UUID:seqno and wsrep_gtid_domain_id:
         cat "$MAGIC_FILE"