MDEV-27902 SIGSEGV's in spider_check_and_set_trx_isolation, spider_db_open_handler, and ha_spider::external_lock and __pthread_mutex_lock: Assertion `mutex->__data.__owner == 0' failed in psi_mutex_lock and inline_mysql_mutex_lock, and SIGABRT in safe_mutex_lockbb-10.10-MDEV-27902

8 files changed, 171 insertions, 3 deletions

diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc
index 533e308dc4e..2db8c57d5a8 100644
--- a/storage/spider/ha_spider.cc
+++ b/storage/spider/ha_spider.cc
@@ -5965,6 +5965,10 @@ int ha_spider::rnd_next(
   int error_num;
   DBUG_ENTER("ha_spider::rnd_next");
   DBUG_PRINT("info",("spider this=%p", this));
+  if ((error_num= spider_alloc_trx_conn_if_freed(this)))
+  {
+    DBUG_RETURN(error_num);
+  }
   if (use_pre_call)
   {
     if (store_error_num)
@@ -12149,6 +12153,11 @@ int ha_spider::lock_tables()
   DBUG_PRINT("info",("spider lock_table_type=%u",
     wide_handler->lock_table_type));
 
+  if ((error_num= spider_alloc_trx_conn_if_freed(this)))
+  {
+    DBUG_RETURN(error_num);
+  }
+
     if (!conns[search_link_idx])
     {
       my_message(ER_SPIDER_REMOTE_SERVER_GONE_AWAY_NUM,
diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_27902.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_27902.result
new file mode 100644
index 00000000000..e3a982300f5
--- /dev/null
+++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_27902.result
@@ -0,0 +1,67 @@
+for master_1
+for child2
+for child3
+connection master_1;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+#
+# MDEV-27902 SIGSEGV's in spider_check_and_set_trx_isolation and
+# spider_db_open_handler, __pthread_mutex_lock:
+# Assertion mutex->__data.__owner == 0' failed in psi_mutex_lock
+# and inline_mysql_mutex_lock
+#
+CREATE TABLE tbl_a (a INT) ENGINE=Spider;
+HANDLER tbl_a OPEN;
+Warnings:
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+HANDLER tbl_a READ FIRST;
+ERROR HY000: Unable to connect to foreign data source: localhost
+HANDLER tbl_a READ NEXT;
+ERROR HY000: Unable to connect to foreign data source: localhost
+DROP TABLE tbl_a;
+#
+# MDEV-28352 Spider: heap-use-after-free in ha_spider::lock_tables(),
+# heap freed by spider_commit()
+#
+CREATE TABLE tbl_a (a INT) ENGINE=SPIDER;
+FLUSH TABLE tbl_a WITH READ LOCK;
+Warnings:
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+BEGIN;
+DROP TABLE tbl_a;
+#
+# MDEV-28676 Spider: Got error 12701 when reading table, when using HANDLER
+#
+CREATE TABLE t (c INT) ENGINE=Spider;
+HANDLER t OPEN;
+Warnings:
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+Error	1429	Unable to connect to foreign data source: localhost
+HANDLER t READ FIRST;
+ERROR HY000: Unable to connect to foreign data source: localhost
+DROP TABLE tbl_a;
+#
+# MDEV-28683 Spider: SIGSEGV in spider_db_direct_delete and SIGSEGV in spider_db_connect
+#
+CREATE TABLE tbl_a (c INT) ENGINE=Spider;
+SELECT * FROM t;
+ERROR HY000: Unable to connect to foreign data source: localhost
+INSERT INTO t (SELECT 1 FROM t);
+ERROR HY000: Unable to connect to foreign data source: localhost
+LOCK TABLES t WRITE CONCURRENT;
+DELETE FROM t;
+ERROR HY000: Unable to connect to foreign data source: localhost
+UNLOCK TABLES;
+DROP TABLE tbl_a;
+DROP DATABASE auto_test_local;
+for master_1
+for child2
+for child3
diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27902.cnf b/storage/spider/mysql-test/spider/bugfix/t/mdev_27902.cnf
new file mode 100644
index 00000000000..b0853e32654
--- /dev/null
+++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27902.cnf
@@ -0,0 +1,2 @@
+!include include/default_mysqld.cnf
+!include ../my_1_1.cnf
diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27902.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_27902.test
new file mode 100644
index 00000000000..86a6b2299b5
--- /dev/null
+++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27902.test
@@ -0,0 +1,63 @@
+--disable_query_log
+--disable_result_log
+--source ../../t/test_init.inc
+--enable_result_log
+--enable_query_log
+
+--connection master_1
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+
+--echo #
+--echo # MDEV-27902 SIGSEGV's in spider_check_and_set_trx_isolation and
+--echo # spider_db_open_handler, __pthread_mutex_lock:
+--echo # Assertion mutex->__data.__owner == 0' failed in psi_mutex_lock
+--echo # and inline_mysql_mutex_lock
+--echo #
+CREATE TABLE tbl_a (a INT) ENGINE=Spider;
+HANDLER tbl_a OPEN;
+--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE
+HANDLER tbl_a READ FIRST;
+--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE
+HANDLER tbl_a READ NEXT;
+DROP TABLE tbl_a;
+
+--echo #
+--echo # MDEV-28352 Spider: heap-use-after-free in ha_spider::lock_tables(),
+--echo # heap freed by spider_commit()
+--echo #
+CREATE TABLE tbl_a (a INT) ENGINE=SPIDER;
+FLUSH TABLE tbl_a WITH READ LOCK;
+BEGIN;
+DROP TABLE tbl_a;
+
+--echo #
+--echo # MDEV-28676 Spider: Got error 12701 when reading table, when using HANDLER
+--echo #
+CREATE TABLE t (c INT) ENGINE=Spider;
+HANDLER t OPEN;
+--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE
+HANDLER t READ FIRST;
+DROP TABLE tbl_a;
+
+--echo #
+--echo # MDEV-28683 Spider: SIGSEGV in spider_db_direct_delete and SIGSEGV in spider_db_connect
+--echo #
+CREATE TABLE tbl_a (c INT) ENGINE=Spider;
+--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE
+SELECT * FROM t;
+--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE
+INSERT INTO t (SELECT 1 FROM t);
+LOCK TABLES t WRITE CONCURRENT;
+--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE
+DELETE FROM t;
+UNLOCK TABLES;
+DROP TABLE tbl_a;
+
+DROP DATABASE auto_test_local;
+
+--disable_query_log
+--disable_result_log
+--source ../../t/test_deinit.inc
+--enable_result_log
+--enable_query_log
diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc
index 366a22ef8d2..b7f706d164e 100644
--- a/storage/spider/spd_conn.cc
+++ b/storage/spider/spd_conn.cc
@@ -288,6 +288,7 @@ void spider_free_conn_from_trx(
   ha_spider *spider;
   SPIDER_IP_PORT_CONN *ip_port_conn = conn->ip_port_conn;
   DBUG_ENTER("spider_free_conn_from_trx");
+
   spider_conn_clear_queue(conn);
   conn->use_for_active_standby = FALSE;
   conn->error_mode = 1;
diff --git a/storage/spider/spd_include.h b/storage/spider/spd_include.h
index 6df79e4e2c8..7c8299e783b 100644
--- a/storage/spider/spd_include.h
+++ b/storage/spider/spd_include.h
@@ -662,13 +662,19 @@ typedef struct st_spider_transaction
   ulonglong          trx_conn_adjustment;
   uint               locked_connections;
 
+  /*
+    conn_hash_freed == TRUE means that elements of trx_conn_hash has been freed
+    and ha_spider::conns maintains freed connections. In such a case, we need
+    to call get_conn() or its wrapper functions before accessing data nodes.
+  */
+  bool conn_hash_freed;
+
   ulonglong          direct_update_count;
   ulonglong          direct_delete_count;
   ulonglong          direct_order_limit_count;
   ulonglong          direct_aggregate_count;
   ulonglong          parallel_search_count;
 
-
   pthread_mutex_t    *udf_table_mutexes;
   CHARSET_INFO       *udf_access_charset;
   spider_string      *udf_set_names;
diff --git a/storage/spider/spd_trx.cc b/storage/spider/spd_trx.cc
index 8facdebb5f3..4cfccf58623 100644
--- a/storage/spider/spd_trx.cc
+++ b/storage/spider/spd_trx.cc
@@ -108,8 +108,12 @@ int spider_free_trx_conn(
       {
         DBUG_ASSERT(!trx_free);
         roop_count++;
-      } else
+      }
+      else
+      {
         spider_free_conn_from_trx(trx, conn, FALSE, trx_free, &roop_count);
+        trx->conn_hash_freed= TRUE;
+      }
     }
     trx->trx_conn_adjustment++;
   } else {
@@ -120,14 +124,28 @@ int spider_free_trx_conn(
       if (conn->table_lock)
       {
         DBUG_ASSERT(!trx_free);
-      } else
+      }
+      else
+      {
         conn->error_mode = 1;
+      }
       roop_count++;
     }
   }
   DBUG_RETURN(0);
 }
 
+int spider_alloc_trx_conn_if_freed(ha_spider *spider)
+{
+  SPIDER_TRX *trx= spider->wide_handler->trx;
+  DBUG_ENTER("spider_alloc_trx_conn_if_freed");
+  if (trx->conn_hash_freed)
+  {
+    DBUG_RETURN(spider_check_trx_and_get_conn(trx->thd, spider, FALSE));
+  }
+  DBUG_RETURN(0);
+}
+
 int spider_free_trx_another_conn(
   SPIDER_TRX *trx,
   bool lock
diff --git a/storage/spider/spd_trx.h b/storage/spider/spd_trx.h
index 2055a49717e..043b8e17594 100644
--- a/storage/spider/spd_trx.h
+++ b/storage/spider/spd_trx.h
@@ -19,6 +19,8 @@ int spider_free_trx_conn(
   bool trx_free
 );
 
+int spider_alloc_trx_conn_if_freed(ha_spider *spider);
+
 int spider_free_trx_another_conn(
   SPIDER_TRX *trx,
   bool lock