MDEV-27282 InnoDB: Failing assertion: !query->intersectionbb-10.3-thiru

- query->intersection fails to get freed if the query exceeds
innodb_ft_result_cache_limit

- errors from init_ftfuncs were not propogated by delete command

This is taken from percona/percona-server@ef2c0bcb9a34aeb06de0058d7c2a2969416b35a7

5 files changed, 116 insertions, 1 deletions

diff --git a/mysql-test/suite/innodb_fts/r/ft_result_cache_limit.result b/mysql-test/suite/innodb_fts/r/ft_result_cache_limit.result
new file mode 100644
index 00000000000..2dbdd5a04bc
--- /dev/null
+++ b/mysql-test/suite/innodb_fts/r/ft_result_cache_limit.result
@@ -0,0 +1,39 @@
+#
+# Bug 1634932: Assertion failure in thread x in
+# file fts0que.cc
+#
+SET @saved_innodb_ft_result_cache_limit= @@global.innodb_ft_result_cache_limit;
+CREATE TABLE `t1` (
+`FTS_DOC_ID` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+`text_content` MEDIUMTEXT, PRIMARY KEY (`FTS_DOC_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+CREATE UNIQUE INDEX FTS_DOC_ID_INDEX ON t1(FTS_DOC_ID);
+SET autocommit=0;
+CREATE PROCEDURE populate_t1()
+BEGIN
+DECLARE i INT DEFAULT 1;
+WHILE (i <= 250) DO
+INSERT INTO t1 (text_content) VALUES ("some_text_1234 aaa");
+SET i = i + 1;
+END WHILE;
+END//
+CALL populate_t1;
+SET autocommit=1;
+SET SESSION debug="+d,fts_instrument_result_cache_limit";
+Warnings:
+Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+ALTER TABLE t1 ADD FULLTEXT INDEX `text_content_idx` (`text_content`);
+SELECT FTS_DOC_ID, text_content
+FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+ERROR HY000: Table handler out of memory
+UPDATE t1
+SET text_content='some_text_12345'
+where MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+ERROR HY000: Table handler out of memory
+DELETE FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+ERROR HY000: Table handler out of memory
+SET GLOBAL innodb_ft_result_cache_limit = @saved_innodb_ft_result_cache_limit;
+DROP TABLE t1;
+DROP PROCEDURE populate_t1;
diff --git a/mysql-test/suite/innodb_fts/t/ft_result_cache_limit.test b/mysql-test/suite/innodb_fts/t/ft_result_cache_limit.test
new file mode 100644
index 00000000000..84254a182d7
--- /dev/null
+++ b/mysql-test/suite/innodb_fts/t/ft_result_cache_limit.test
@@ -0,0 +1,57 @@
+--echo #
+--echo # Bug 1634932: Assertion failure in thread x in
+--echo # file fts0que.cc
+--echo #
+
+--source include/have_innodb.inc
+--source include/have_debug.inc
+
+SET @saved_innodb_ft_result_cache_limit= @@global.innodb_ft_result_cache_limit;
+
+CREATE TABLE `t1` (
+	`FTS_DOC_ID` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+	`text_content` MEDIUMTEXT, PRIMARY KEY (`FTS_DOC_ID`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+CREATE UNIQUE INDEX FTS_DOC_ID_INDEX ON t1(FTS_DOC_ID);
+
+SET autocommit=0;
+
+DELIMITER //;
+CREATE PROCEDURE populate_t1()
+BEGIN
+        DECLARE i INT DEFAULT 1;
+        WHILE (i <= 250) DO
+          INSERT INTO t1 (text_content) VALUES ("some_text_1234 aaa");
+          SET i = i + 1;
+        END WHILE;
+END//
+
+DELIMITER ;//
+
+CALL populate_t1;
+SET autocommit=1;
+
+SET SESSION debug="+d,fts_instrument_result_cache_limit";
+
+ALTER TABLE t1 ADD FULLTEXT INDEX `text_content_idx` (`text_content`);
+
+# HA_ERR_FTS_EXCEED_RESULT_CACHE_LIMIT = 188
+--error 128
+SELECT FTS_DOC_ID, text_content
+FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+
+--error 128
+UPDATE t1
+SET text_content='some_text_12345'
+where MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+
+--error 128
+DELETE FROM t1
+WHERE MATCH text_content AGAINST ('+some_text_1234' IN BOOLEAN MODE);
+
+SET GLOBAL innodb_ft_result_cache_limit = @saved_innodb_ft_result_cache_limit;
+
+DROP TABLE t1;
+DROP PROCEDURE populate_t1;
diff --git a/sql/item_func.cc b/sql/item_func.cc
index 3fd69405456..2c79ca531d3 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -6012,6 +6012,8 @@ bool Item_func_match::init_search(THD *thd, bool no_order)
 
   ft_handler= table->file->ft_init_ext(flags, key, ft_tmp);
 
+  if (!ft_handler)
+    DBUG_RETURN(1);
   if (join_key)
     table->file->ft_handler=ft_handler;
 
diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index a0d8feb61e8..95de5d6ad61 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -1184,7 +1184,9 @@ multi_delete::initialize_tables(JOIN *join)
                                                   table->file->ref_length,
                                                   MEM_STRIP_BUF_SIZE);
   }
-  init_ftfuncs(thd, thd->lex->current_select, 1);
+  if (init_ftfuncs(thd, thd->lex->current_select, 1))
+    DBUG_RETURN(true);
+
   DBUG_RETURN(thd->is_fatal_error);
 }
 
diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc
index 01227a8df58..f7fc543e211 100644
--- a/storage/innobase/fts/fts0que.cc
+++ b/storage/innobase/fts/fts0que.cc
@@ -1202,6 +1202,14 @@ fts_query_difference(
 	return(query->error);
 }
 
+/* Free the query intersection
+@param	query	query instance */
+static void fts_query_free_intersection(fts_query_t* query)
+{
+  fts_query_free_doc_ids(query, query->intersection);
+  query->intersection = NULL;
+}
+
 /*****************************************************************//**
 Intersect the token doc ids with the current set.
 @return DB_SUCCESS if all go well */
@@ -1300,6 +1308,7 @@ fts_query_intersect(
 		/* error is passed by 'query->error' */
 		if (query->error != DB_SUCCESS) {
 			ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT);
+			fts_query_free_intersection(query);
 			return(query->error);
 		}
 
@@ -1328,6 +1337,8 @@ fts_query_intersect(
 
 			ut_a(!query->multi_exist || (query->multi_exist
 			     && rbt_size(query->doc_ids) <= n_doc_ids));
+		} else if (query->intersection) {
+			fts_query_free_intersection(query);
 		}
 	}
 
@@ -1546,6 +1557,10 @@ fts_merge_doc_ids(
 				query, ranking->doc_id, ranking->rank);
 
 		if (query->error != DB_SUCCESS) {
+			if (query->intersection) {
+				ut_a(query->oper == FTS_EXIST);
+				fts_query_free_intersection(query);
+			}
 			DBUG_RETURN(query->error);
 		}