MDEV-28641 : Query cache entries not invalidated on slave of a Galera clusterbb-10.4-MDEV-28641-galera

Query cache should be invalidated if we are not in applier. For some
reason this condition was incorrect starting from 10.5 but it is
correct in 10.4.

Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

6 files changed, 344 insertions, 2 deletions

diff --git a/mysql-test/suite/galera/galera_2x2nodes.cnf b/mysql-test/suite/galera/galera_2x2nodes.cnf
new file mode 100644
index 00000000000..922906eac6f
--- /dev/null
+++ b/mysql-test/suite/galera/galera_2x2nodes.cnf
@@ -0,0 +1,77 @@
+# Use default setting for mysqld processes
+!include include/default_mysqld.cnf
+
+[mysqld]
+loose-innodb
+log-bin=mysqld-bin
+log-slave-updates
+binlog-format=row
+innodb-autoinc-lock-mode=2
+default-storage-engine=innodb
+wsrep_gtid_mode=1
+gtid_ignore_duplicates
+auto_increment_increment=3
+
+wsrep-provider=@ENV.WSREP_PROVIDER
+# enforce read-committed characteristics across the cluster
+# wsrep-causal-reads=ON
+wsrep-sync-wait=15
+
+[mysqld.1]
+wsrep-on=1
+#galera_port=@OPT.port
+#ist_port=@OPT.port
+#sst_port=@OPT.port
+wsrep_cluster_address=gcomm://
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+wsrep_node_address='127.0.0.1:@mysqld.1.#galera_port'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
+
+[mysqld.2]
+wsrep-on=1
+#galera_port=@OPT.port
+#ist_port=@OPT.port
+#sst_port=@OPT.port
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+wsrep_node_address='127.0.0.1:@mysqld.2.#galera_port'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+
+[mysqld.3]
+wsrep-on=1
+#galera_port=@OPT.port
+#ist_port=@OPT.port
+#sst_port=@OPT.port
+wsrep_cluster_address=gcomm://
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+wsrep_node_address='127.0.0.1:@mysqld.3.#galera_port'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'
+
+[mysqld.4]
+wsrep-on=1
+#galera_port=@OPT.port
+#ist_port=@OPT.port
+#sst_port=@OPT.port
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.3.#galera_port'
+wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.4.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
+wsrep_node_address='127.0.0.1:@mysqld.4.#galera_port'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.4.#sst_port'
+
+[ENV]
+NODE_MYPORT_1= @mysqld.1.port
+NODE_MYSOCK_1= @mysqld.1.socket
+
+NODE_MYPORT_2= @mysqld.2.port
+NODE_MYSOCK_2= @mysqld.2.socket
+
+NODE_MYPORT_3= @mysqld.3.port
+NODE_MYSOCK_3= @mysqld.3.socket
+
+NODE_MYPORT_4= @mysqld.4.port
+NODE_MYSOCK_4= @mysqld.4.socket
+
+
diff --git a/mysql-test/suite/galera/r/galera_query_cache_invalidate.result b/mysql-test/suite/galera/r/galera_query_cache_invalidate.result
new file mode 100644
index 00000000000..98438b3b527
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_query_cache_invalidate.result
@@ -0,0 +1,119 @@
+connection node_2;
+connection node_1;
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+connection node_2;
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
+connection node_4;
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
+connection node_3;
+CHANGE MASTER TO master_host='127.0.0.1', master_user='root', master_port=NODE_MYPORT_1, master_use_gtid=current_pos;;
+START SLAVE;
+include/wait_for_slave_to_start.inc
+connection node_1;
+CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb;
+SET AUTOCOMMIT=1;
+INSERT INTO t1 VALUES (4000000, 'foobar');
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+50001
+connection node_3;
+connection node_1;
+# node_1
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+connection node_2;
+# node_2
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+connection node_3;
+# node_3
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+connection node_4;
+# node_4
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+connection node_1;
+# node_1 insert new
+INSERT INTO t1 values (5000000, 'foobar');
+connection node_3;
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+connection node_2;
+# node_2
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+connection node_3;
+# node_3
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+connection node_4;
+# node_4
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+SELECT * FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+connection node_2;
+# node_3 different query same table
+SELECT id, msg FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+connection node_4;
+# node_6 different query same table
+SELECT id, msg FROM t1 WHERE msg='foobar';
+id	msg
+4000000	foobar
+5000000	foobar
+connection node_1;
+drop table t1;
+connection node_3;
+connection node_3;
+STOP SLAVE;
+RESET SLAVE ALL;
+connection node_1;
+SET SESSION WSREP_ON=OFF;
+RESET MASTER;
+SET SESSION WSREP_ON=ON;
+disconnect node_2;
+disconnect node_1;
+# End of test
diff --git a/mysql-test/suite/galera/t/galera_query_cache_invalidate.cnf b/mysql-test/suite/galera/t/galera_query_cache_invalidate.cnf
new file mode 100644
index 00000000000..49269422a35
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_query_cache_invalidate.cnf
@@ -0,0 +1,25 @@
+!include ../galera_2x2nodes.cnf
+
+[mysqld.1]
+wsrep_gtid_domain_id=1
+server-id=11
+query_cache_type=1
+query_cache_size=15M
+
+[mysqld.2]
+wsrep_gtid_domain_id=1
+server-id=12
+query_cache_type=1
+query_cache_size=15M
+
+[mysqld.3]
+wsrep_gtid_domain_id=2
+server-id=13
+query_cache_type=1
+query_cache_size=15M
+
+[mysqld.4]
+wsrep_gtid_domain_id=2
+server-id=21
+query_cache_type=1
+query_cache_size=15M
diff --git a/mysql-test/suite/galera/t/galera_query_cache_invalidate.test b/mysql-test/suite/galera/t/galera_query_cache_invalidate.test
new file mode 100644
index 00000000000..d72d8a9ba40
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_query_cache_invalidate.test
@@ -0,0 +1,120 @@
+--source include/big_test.inc
+--source include/force_restart.inc
+--source include/galera_cluster.inc
+--source include/have_sequence.inc
+
+#
+# MDEV-28641 : Query cache entries not invalidated on slave of a Galera cluster
+#
+# We use two 2-node galera clusters as follows
+#
+#       A(1) <-> B(2)              {Galera cluster 1}
+#       |                          {Async replication}
+#       D(3) <-> E(4)              {Galera cluster 2}
+#
+# Normal asyncronous replication is used between nodes 1 and 3
+# so that node_1 is master and node_3 a slave.
+#
+# In this test we can't test is some query fast or slow but we can
+# test does all nodes see all rows (this is not true before fix)
+#
+--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
+--connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4
+
+--connection node_2
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
+--connection node_4
+call mtr.add_suppression("WSREP: Ignoring server id for non bootstrap node.");
+
+--connection node_3
+
+--replace_result $NODE_MYPORT_1 NODE_MYPORT_1
+--eval CHANGE MASTER TO master_host='127.0.0.1', master_user='root', master_port=$NODE_MYPORT_1, master_use_gtid=current_pos;
+START SLAVE;
+--source include/wait_for_slave_to_start.inc
+
+--connection node_1
+
+CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb;
+--disable_query_log
+INSERT INTO t1 SELECT seq, md5(rand()) from seq_1_to_50000;
+COMMIT;
+--enable_query_log
+SET AUTOCOMMIT=1;
+INSERT INTO t1 VALUES (4000000, 'foobar');
+SELECT COUNT(*) FROM t1;
+--sync_slave_with_master node_3
+
+#
+# All nodes should see one row and first query is slow and second fast
+#
+--connection node_1
+--echo # node_1
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+--connection node_2
+--echo # node_2
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+--connection node_3
+--echo # node_3
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+--connection node_4
+--echo # node_4
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+#
+# Insert a new row in master, this should cause query cache
+# invalidation
+#
+--connection node_1
+--echo # node_1 insert new
+INSERT INTO t1 values (5000000, 'foobar');
+--sync_slave_with_master node_3
+
+#
+# All nodes should see 2 rows
+#
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+
+--connection node_2
+--echo # node_2
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+--connection node_3
+--echo # node_3
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+--connection node_4
+--echo # node_4
+SELECT * FROM t1 WHERE msg='foobar';
+SELECT * FROM t1 WHERE msg='foobar';
+
+--connection node_2
+--echo # node_3 different query same table
+SELECT id, msg FROM t1 WHERE msg='foobar';
+
+--connection node_4
+--echo # node_6 different query same table
+SELECT id, msg FROM t1 WHERE msg='foobar';
+
+#
+# Cleanup
+#
+--connection node_1
+drop table t1;
+--sync_slave_with_master node_3
+
+--connection node_3
+STOP SLAVE;
+RESET SLAVE ALL;
+
+--connection node_1
+SET SESSION WSREP_ON=OFF;
+RESET MASTER;
+SET SESSION WSREP_ON=ON;
+
+--source include/galera_end.inc
+--echo # End of test
diff --git a/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf b/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf
index 9f99adbd711..cd7a892f4c9 100644
--- a/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf
+++ b/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf
@@ -55,7 +55,7 @@ wsrep-on=1
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
-wsrep_cluster_address='gcomm://
+wsrep_cluster_address=gcomm://
 wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.4.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;pc.wait_prim_timeout=PT60S;gcache.size=10M'
 wsrep_node_address='127.0.0.1:@mysqld.4.#galera_port'
 wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port
diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc
index 56d1c978ac4..c5fb637a000 100644
--- a/sql/log_event_server.cc
+++ b/sql/log_event_server.cc
@@ -5586,7 +5586,8 @@ int Rows_log_event::do_apply_event(rpl_group_info *rgi)
       to avoid query cache being polluted with stale entries,
     */
 # ifdef WITH_WSREP
-    if (!WSREP(thd) && !wsrep_thd_is_applying(thd))
+    /* Query cache is not invalidated on wsrep applier here */
+    if (!(WSREP(thd) && wsrep_thd_is_applying(thd)))
 # endif /* WITH_WSREP */
       query_cache.invalidate_locked_for_write(thd, rgi->tables_to_lock);
 #endif /* HAVE_QUERY_CACHE */