MDEV-27862 Galera should replicate nextval()-related changes in sequences with INCREMENT <> 0, at least NOCACHE ones with engine=InnoDB

Sequence storage engine is not transactionl so cache will be written in
stmt_cache that is not replicated in cluster. To fix this replicate
what is available in both trans_cache and stmt_cache.

Sequences will only work when NOCACHE keyword is used when sequnce is
created. If WSREP is enabled and we don't have this keyword report error
indicting that sequence will not work correctly in cluster.

When binlog is enabled statement cache will be cleared in transaction
before COMMIT so cache generated from sequence will not be replicated.
We need to keep cache until replication.

Tests are re-recorded because of replication changes that were
introducted with this PR.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

13 files changed, 197 insertions, 13 deletions

diff --git a/mysql-test/suite/galera/r/MDEV-18832.result b/mysql-test/suite/galera/r/MDEV-18832.result
index 700a0bbefb1..2e0872b9f2e 100644
--- a/mysql-test/suite/galera/r/MDEV-18832.result
+++ b/mysql-test/suite/galera/r/MDEV-18832.result
@@ -12,3 +12,6 @@ INSERT INTO t1 VALUES (NEXT VALUE FOR Seq1_1);
 ERROR 23000: Duplicate entry '1' for key 'PRIMARY'
 DROP SEQUENCE Seq1_1;
 DROP TABLE t1;
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+connection node_2;
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
diff --git a/mysql-test/suite/galera/r/MDEV-27862.result b/mysql-test/suite/galera/r/MDEV-27862.result
new file mode 100644
index 00000000000..25b7bc6cfd2
--- /dev/null
+++ b/mysql-test/suite/galera/r/MDEV-27862.result
@@ -0,0 +1,54 @@
+connection node_2;
+connection node_1;
+CREATE SEQUENCE seq_nocache ENGINE=InnoDB;
+DROP SEQUENCE seq_nocache;
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+connection node_2;
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+connection node_1;
+CREATE SEQUENCE seq NOCACHE ENGINE=InnoDB;
+SELECT NEXTVAL(seq) = 1;
+NEXTVAL(seq) = 1
+1
+connection node_2;
+SELECT NEXTVAL(seq) = 2;
+NEXTVAL(seq) = 2
+1
+connection node_1;
+SELECT NEXTVAL(seq) = 3;
+NEXTVAL(seq) = 3
+1
+SELECT SETVAL(seq, 100);
+SETVAL(seq, 100)
+100
+connection node_2;
+SELECT NEXTVAL(seq) = 101;
+NEXTVAL(seq) = 101
+1
+connection node_1;
+SELECT NEXTVAL(seq) = 102;
+NEXTVAL(seq) = 102
+1
+DROP SEQUENCE seq;
+CREATE TABLE t1(f1 INT);
+CREATE SEQUENCE seq_transaction NOCACHE ENGINE=InnoDB;
+START TRANSACTION;
+INSERT INTO t1 VALUES (0);
+SELECT NEXTVAL(seq_transaction);
+NEXTVAL(seq_transaction)
+1
+INSERT INTO t1 VALUES (NEXTVAL(seq_transaction));
+COMMIT;
+connection node_2;
+SELECT COUNT(*) = 2 FROM t1;
+COUNT(*) = 2
+1
+SELECT NEXTVAL(seq_transaction) = 3;
+NEXTVAL(seq_transaction) = 3
+1
+connection node_1;
+SELECT NEXTVAL(seq_transaction) = 4;
+NEXTVAL(seq_transaction) = 4
+1
+DROP SEQUENCE seq_transaction;
+DROP TABLE t1;
diff --git a/mysql-test/suite/galera/r/galera_sequences.result b/mysql-test/suite/galera/r/galera_sequences.result
index 48593d2a258..7276cb8dbde 100644
--- a/mysql-test/suite/galera/r/galera_sequences.result
+++ b/mysql-test/suite/galera/r/galera_sequences.result
@@ -44,6 +44,9 @@ Table	Create Table
 Seq1_1	CREATE SEQUENCE `Seq1_1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB
 select NEXT VALUE FOR Seq1_1;
 NEXT VALUE FOR Seq1_1
-1
+3001
 connection node_1;
 DROP SEQUENCE Seq1_1;
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+connection node_2;
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
diff --git a/mysql-test/suite/galera/t/MDEV-18832.test b/mysql-test/suite/galera/t/MDEV-18832.test
index e2f1b2afd45..ba93761435a 100644
--- a/mysql-test/suite/galera/t/MDEV-18832.test
+++ b/mysql-test/suite/galera/t/MDEV-18832.test
@@ -13,3 +13,11 @@ CREATE SEQUENCE Seq1_1  START WITH 1  INCREMENT BY 1;
 INSERT INTO t1 VALUES (NEXT VALUE FOR Seq1_1);
 DROP SEQUENCE Seq1_1;
 DROP TABLE t1;
+
+# Supress warning for SEQUENCES that are declared without `NOCACHE` introduced with MDEV-27862
+
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+
+--connection node_2
+
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
diff --git a/mysql-test/suite/galera/t/MDEV-27862.combinations b/mysql-test/suite/galera/t/MDEV-27862.combinations
new file mode 100644
index 00000000000..1eeb8fb4614
--- /dev/null
+++ b/mysql-test/suite/galera/t/MDEV-27862.combinations
@@ -0,0 +1,4 @@
+[binlogoff]
+
+[binlogon]
+log-bin
diff --git a/mysql-test/suite/galera/t/MDEV-27862.test b/mysql-test/suite/galera/t/MDEV-27862.test
new file mode 100644
index 00000000000..89d3465b91f
--- /dev/null
+++ b/mysql-test/suite/galera/t/MDEV-27862.test
@@ -0,0 +1,67 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+# Report WARNING when SEQUENCE is created without `NOCACHE`
+
+CREATE SEQUENCE seq_nocache ENGINE=InnoDB;
+DROP SEQUENCE seq_nocache;
+
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+
+--connection node_2
+
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+
+# NEXTVAL
+
+--connection node_1
+
+CREATE SEQUENCE seq NOCACHE ENGINE=InnoDB;
+
+SELECT NEXTVAL(seq) = 1;
+
+--connection node_2
+
+SELECT NEXTVAL(seq) = 2;
+
+--connection node_1
+
+SELECT NEXTVAL(seq) = 3;
+
+
+# SETVAL
+
+SELECT SETVAL(seq, 100);
+
+--connection node_2
+
+SELECT NEXTVAL(seq) = 101;
+
+--connection node_1
+
+SELECT NEXTVAL(seq) = 102;
+
+DROP SEQUENCE seq;
+
+# TRANSACTIONS
+
+CREATE TABLE t1(f1 INT);
+CREATE SEQUENCE seq_transaction NOCACHE ENGINE=InnoDB;
+
+START TRANSACTION;
+INSERT INTO t1 VALUES (0);
+SELECT NEXTVAL(seq_transaction);
+INSERT INTO t1 VALUES (NEXTVAL(seq_transaction));
+COMMIT;
+
+--connection node_2
+
+SELECT COUNT(*) = 2 FROM t1;
+SELECT NEXTVAL(seq_transaction) = 3;
+
+--connection node_1
+SELECT NEXTVAL(seq_transaction) = 4;
+
+DROP SEQUENCE seq_transaction;
+DROP TABLE t1;
+
diff --git a/mysql-test/suite/galera/t/galera_sequences.test b/mysql-test/suite/galera/t/galera_sequences.test
index 480366f6a6f..d469cc73516 100644
--- a/mysql-test/suite/galera/t/galera_sequences.test
+++ b/mysql-test/suite/galera/t/galera_sequences.test
@@ -44,3 +44,9 @@ select NEXT VALUE FOR Seq1_1;
 
 --connection node_1
 DROP SEQUENCE Seq1_1;
+
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+
+--connection node_2
+
+CALL mtr.add_suppression("SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
diff --git a/sql/ha_sequence.cc b/sql/ha_sequence.cc
index b0611c1505a..1331fea74d1 100644
--- a/sql/ha_sequence.cc
+++ b/sql/ha_sequence.cc
@@ -434,6 +434,9 @@ static int sequence_initialize(void *p)
                                HTON_HIDDEN |
                                HTON_TEMPORARY_NOT_SUPPORTED |
                                HTON_ALTER_NOT_SUPPORTED |
+#ifdef WITH_WSREP
+                               HTON_WSREP_REPLICATION |
+#endif
                                HTON_NO_PARTITION);
   DBUG_RETURN(0);
 }
diff --git a/sql/log.cc b/sql/log.cc
index 25cf0831469..ec96a2f9b23 100644
--- a/sql/log.cc
+++ b/sql/log.cc
@@ -2013,7 +2013,13 @@ static int binlog_commit(handlerton *hton, THD *thd, bool all)
 
   thd->backup_stage(&org_stage);
   THD_STAGE_INFO(thd, stage_binlog_write);
+#ifdef WITH_WSREP
+  // DON'T clear stmt cache in case we are in transaction
+  if (!cache_mngr->stmt_cache.empty() &&
+      (!wsrep_on(thd) || ending_trans(thd, all)))
+#else
   if (!cache_mngr->stmt_cache.empty())
+#endif
   {
     error= binlog_commit_flush_stmt_cache(thd, all, cache_mngr);
   }
@@ -10777,13 +10783,13 @@ maria_declare_plugin_end;
 #ifdef WITH_WSREP
 #include "wsrep_mysqld.h"
 
-IO_CACHE *wsrep_get_trans_cache(THD * thd)
+IO_CACHE *wsrep_get_cache(THD * thd, bool is_transactional)
 {
   DBUG_ASSERT(binlog_hton->slot != HA_SLOT_UNDEF);
   binlog_cache_mngr *cache_mngr = (binlog_cache_mngr*)
     thd_get_ha_data(thd, binlog_hton);
   if (cache_mngr)
-    return cache_mngr->get_binlog_cache_log(true);
+    return cache_mngr->get_binlog_cache_log(is_transactional);
 
   WSREP_DEBUG("binlog cache not initialized, conn: %llu",
 	      thd->thread_id);
diff --git a/sql/log.h b/sql/log.h
index 5cb75c97b5e..2520c8fede8 100644
--- a/sql/log.h
+++ b/sql/log.h
@@ -1239,7 +1239,7 @@ static inline TC_LOG *get_tc_log_implementation()
 }
 
 #ifdef WITH_WSREP
-IO_CACHE* wsrep_get_trans_cache(THD *);
+IO_CACHE* wsrep_get_cache(THD *, bool);
 void wsrep_thd_binlog_trx_reset(THD * thd);
 void wsrep_thd_binlog_stmt_rollback(THD * thd);
 #endif /* WITH_WSREP */
diff --git a/sql/sql_sequence.cc b/sql/sql_sequence.cc
index 367fbad144c..8488fa67ecb 100644
--- a/sql/sql_sequence.cc
+++ b/sql/sql_sequence.cc
@@ -308,6 +308,11 @@ bool sequence_insert(THD *thd, LEX *lex, TABLE_LIST *org_table_list)
       DBUG_RETURN(TRUE);
   }
 
+#ifdef WITH_WSREP
+  if (WSREP_ON && seq->cache != 0)
+    WSREP_WARN("CREATE SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+#endif
+
   /* If not temporary table */
   if (!temporary_table)
   {
@@ -904,12 +909,18 @@ bool Sql_cmd_alter_sequence::execute(THD *thd)
   No_such_table_error_handler no_such_table_handler;
   DBUG_ENTER("Sql_cmd_alter_sequence::execute");
 
+
   if (check_access(thd, ALTER_ACL, first_table->db.str,
                    &first_table->grant.privilege,
                    &first_table->grant.m_internal,
                    0, 0))
     DBUG_RETURN(TRUE);                  /* purecov: inspected */
 
+#ifdef WITH_WSREP
+  if (WSREP_ON && new_seq->cache != 0)
+    WSREP_WARN("ALTER SEQUENCES declared without `NOCACHE` will not behave correctly in galera cluster.");
+#endif
+
   if (check_grant(thd, ALTER_ACL, first_table, FALSE, 1, FALSE))
     DBUG_RETURN(TRUE);                  /* purecov: inspected */
 
diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc
index d04d9989e99..5c1c899fcde 100644
--- a/sql/wsrep_binlog.cc
+++ b/sql/wsrep_binlog.cc
@@ -155,10 +155,10 @@ static int wsrep_write_cache_inc(THD*      const thd,
         goto cleanup;
       cache->read_pos= cache->read_end;
     } while ((cache->file >= 0) && (length= my_b_fill(cache)));
-  }
-  if (ret == 0)
-  {
-    assert(total_length + thd->wsrep_sr().log_position() == saved_pos);
+    if (ret == 0)
+    {
+      assert(total_length + thd->wsrep_sr().log_position() == saved_pos);
+    }
   }
 
 cleanup:
diff --git a/sql/wsrep_client_service.cc b/sql/wsrep_client_service.cc
index 464296ea6cf..59ba8c30e43 100644
--- a/sql/wsrep_client_service.cc
+++ b/sql/wsrep_client_service.cc
@@ -86,18 +86,37 @@ int Wsrep_client_service::prepare_data_for_replication()
   DBUG_ASSERT(m_thd == current_thd);
   DBUG_ENTER("Wsrep_client_service::prepare_data_for_replication");
   size_t data_len= 0;
-  IO_CACHE* cache= wsrep_get_trans_cache(m_thd);
+  IO_CACHE* transactional_cache= wsrep_get_cache(m_thd, true);
+  IO_CACHE* stmt_cache= wsrep_get_cache(m_thd, false);
 
-  if (cache)
+  if (transactional_cache || stmt_cache)
   {
     m_thd->binlog_flush_pending_rows_event(true);
-    if (wsrep_write_cache(m_thd, cache, &data_len))
+
+    size_t transactional_data_len= 0;
+    size_t stmt_data_len= 0;
+
+    // Write transactional cache
+    if (transactional_cache &&
+        wsrep_write_cache(m_thd, transactional_cache, &transactional_data_len))
     {
       WSREP_ERROR("rbr write fail, data_len: %zu",
                   data_len);
       // wsrep_override_error(m_thd, ER_ERROR_DURING_COMMIT);
       DBUG_RETURN(1);
     }
+
+    // Write stmt cache
+    if (stmt_cache && wsrep_write_cache(m_thd, stmt_cache, &stmt_data_len))
+    {
+      WSREP_ERROR("rbr write fail, data_len: %zu",
+                  data_len);
+      // wsrep_override_error(m_thd, ER_ERROR_DURING_COMMIT);
+      DBUG_RETURN(1);
+    }
+
+    // Complete data written from both caches
+    data_len = transactional_data_len + stmt_data_len;
   }
 
   if (data_len == 0)
@@ -139,7 +158,7 @@ int Wsrep_client_service::prepare_fragment_for_replication(
   DBUG_ASSERT(m_thd == current_thd);
   THD* thd= m_thd;
   DBUG_ENTER("Wsrep_client_service::prepare_fragment_for_replication");
-  IO_CACHE* cache= wsrep_get_trans_cache(thd);
+  IO_CACHE* cache= wsrep_get_cache(thd, true);
   thd->binlog_flush_pending_rows_event(true);
 
   if (!cache)
@@ -221,7 +240,7 @@ bool Wsrep_client_service::statement_allowed_for_streaming() const
 
 size_t Wsrep_client_service::bytes_generated() const
 {
-  IO_CACHE* cache= wsrep_get_trans_cache(m_thd);
+  IO_CACHE* cache= wsrep_get_cache(m_thd, true);
   if (cache)
   {
     size_t pending_rows_event_length= 0;