MDEV-24088 Assertion in InnoDB's FTS code may be triggered by a repeated words fed to simple_parser pluginbb-10.2-MDEV-24088

- InnoDB simple parser fails while parsing the repeated word.
This failure mainly because the simple parser fails to
update the position of the keyword within the word. The patch
is basically use the mysql/mysql-server@0c954c2289a75d90d1088356b1092437ebf45a1d
to solve the issue.

12 files changed, 45 insertions, 16 deletions

diff --git a/include/mysql/plugin_audit.h.pp b/include/mysql/plugin_audit.h.pp
index 410781669ba..31ce06ceb92 100644
--- a/include/mysql/plugin_audit.h.pp
+++ b/include/mysql/plugin_audit.h.pp
@@ -453,6 +453,7 @@ typedef struct st_mysql_ftparser_boolean_info
   int weight_adjust;
   char wasign;
   char trunc;
+  int position;
   char prev;
   char *quot;
 } MYSQL_FTPARSER_BOOLEAN_INFO;
diff --git a/include/mysql/plugin_auth.h.pp b/include/mysql/plugin_auth.h.pp
index 62061f74274..76b8dfe4c59 100644
--- a/include/mysql/plugin_auth.h.pp
+++ b/include/mysql/plugin_auth.h.pp
@@ -453,6 +453,7 @@ typedef struct st_mysql_ftparser_boolean_info
   int weight_adjust;
   char wasign;
   char trunc;
+  int position;
   char prev;
   char *quot;
 } MYSQL_FTPARSER_BOOLEAN_INFO;
diff --git a/include/mysql/plugin_encryption.h.pp b/include/mysql/plugin_encryption.h.pp
index 1277147e55f..b0a98e294eb 100644
--- a/include/mysql/plugin_encryption.h.pp
+++ b/include/mysql/plugin_encryption.h.pp
@@ -453,6 +453,7 @@ typedef struct st_mysql_ftparser_boolean_info
   int weight_adjust;
   char wasign;
   char trunc;
+  int position;
   char prev;
   char *quot;
 } MYSQL_FTPARSER_BOOLEAN_INFO;
diff --git a/include/mysql/plugin_ftparser.h b/include/mysql/plugin_ftparser.h
index 8db8712926f..2c35ee94cc8 100644
--- a/include/mysql/plugin_ftparser.h
+++ b/include/mysql/plugin_ftparser.h
@@ -26,7 +26,7 @@ extern "C" {
   API for Full-text parser plugin. (MYSQL_FTPARSER_PLUGIN)
 */
 
-#define MYSQL_FTPARSER_INTERFACE_VERSION 0x0100
+#define MYSQL_FTPARSER_INTERFACE_VERSION 0x0101
 
 /* Parsing modes. Set in  MYSQL_FTPARSER_PARAM::mode */
 enum enum_ftparser_mode
@@ -115,6 +115,9 @@ enum enum_ft_token_type
 
   trunc: Corresponds to the '*' operator in the default setting of the
   ft_boolean_syntax system variable.
+
+  position: Start position in bytes of the word in the document,
+  used by InnoDB FTS
 */
 
 typedef struct st_mysql_ftparser_boolean_info
@@ -124,6 +127,7 @@ typedef struct st_mysql_ftparser_boolean_info
   int weight_adjust;
   char wasign;
   char trunc;
+  int position;
   /* These are parser state and must be removed. */
   char prev;
   char *quot;
diff --git a/include/mysql/plugin_ftparser.h.pp b/include/mysql/plugin_ftparser.h.pp
index 85c6700227f..06e6c946902 100644
--- a/include/mysql/plugin_ftparser.h.pp
+++ b/include/mysql/plugin_ftparser.h.pp
@@ -496,6 +496,7 @@ typedef struct st_mysql_ftparser_boolean_info
   int weight_adjust;
   char wasign;
   char trunc;
+  int position;
   char prev;
   char *quot;
 } MYSQL_FTPARSER_BOOLEAN_INFO;
diff --git a/include/mysql/plugin_password_validation.h.pp b/include/mysql/plugin_password_validation.h.pp
index 25eb5425e52..587b454ecc8 100644
--- a/include/mysql/plugin_password_validation.h.pp
+++ b/include/mysql/plugin_password_validation.h.pp
@@ -453,6 +453,7 @@ typedef struct st_mysql_ftparser_boolean_info
   int weight_adjust;
   char wasign;
   char trunc;
+  int position;
   char prev;
   char *quot;
 } MYSQL_FTPARSER_BOOLEAN_INFO;
diff --git a/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result b/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result
index b7688e9ef0f..fc60e134b1f 100644
--- a/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result
+++ b/mysql-test/suite/innodb_fts/r/innodb_fts_plugin.result
@@ -213,3 +213,17 @@ ERROR HY000: Function 'simple_parser' is not defined
 DROP TABLE articles;
 UNINSTALL PLUGIN simple_parser;
 ERROR 42000: PLUGIN simple_parser does not exist
+#
+# MDEV-24088 Assertion in InnoDB's FTS code may be
+# triggered by a repeated words fed to
+# simple_parser plugin
+#
+INSTALL PLUGIN simple_parser SONAME 'mypluglib';
+CREATE TABLE articles(
+a TEXT DEFAULT NULL,
+b TEXT DEFAULT NULL,
+FULLTEXT (a, b) WITH PARSER simple_parser
+) ENGINE=InnoDB;
+INSERT INTO articles VALUES ('111', '1234 1234 1234');
+DROP TABLE articles;
+UNINSTALL PLUGIN simple_parser;
diff --git a/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test b/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test
index cd31500b23f..6d54913006f 100644
--- a/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test
+++ b/mysql-test/suite/innodb_fts/t/innodb_fts_plugin.test
@@ -213,6 +213,22 @@ CREATE TABLE articles2 (
 	) ENGINE=InnoDB;
 
 DROP TABLE articles;
+
 # Uninstall plugin
 -- error ER_SP_DOES_NOT_EXIST
 UNINSTALL PLUGIN simple_parser;
+
+--echo #
+--echo # MDEV-24088 Assertion in InnoDB's FTS code may be
+--echo # triggered by a repeated words fed to
+--echo # simple_parser plugin
+--echo #
+INSTALL PLUGIN simple_parser SONAME 'mypluglib';
+CREATE TABLE articles(
+ a TEXT DEFAULT NULL,
+ b TEXT DEFAULT NULL,
+ FULLTEXT (a, b) WITH PARSER simple_parser
+) ENGINE=InnoDB;
+INSERT INTO articles VALUES ('111', '1234 1234 1234');
+DROP TABLE articles;
+UNINSTALL PLUGIN simple_parser;
diff --git a/plugin/fulltext/plugin_example.c b/plugin/fulltext/plugin_example.c
index fb8de478a9f..238daa9d678 100644
--- a/plugin/fulltext/plugin_example.c
+++ b/plugin/fulltext/plugin_example.c
@@ -148,7 +148,7 @@ static int simple_parser_deinit(MYSQL_FTPARSER_PARAM *param
 static void add_word(MYSQL_FTPARSER_PARAM *param, const char *word, size_t len)
 {
   MYSQL_FTPARSER_BOOLEAN_INFO bool_info=
-    { FT_TOKEN_WORD, 0, 0, 0, 0, ' ', 0 };
+    { FT_TOKEN_WORD, 0, 0, 0, 0, (word - param->doc), ' ', 0 };
 
   param->mysql_add_word(param, word, (int)len, &bool_info);
 }
diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
index b423db71c7d..06bf1911dd3 100644
--- a/storage/innobase/fts/fts0fts.cc
+++ b/storage/innobase/fts/fts0fts.cc
@@ -4617,12 +4617,8 @@ fts_tokenize_document_internal(
 {
 	fts_string_t	str;
 	byte		buf[FTS_MAX_WORD_LEN + 1];
-	/* JAN: TODO: MySQL 5.7
 	MYSQL_FTPARSER_BOOLEAN_INFO bool_info =
 		{ FT_TOKEN_WORD, 0, 0, 0, 0, 0, ' ', 0 };
-	*/
-	MYSQL_FTPARSER_BOOLEAN_INFO bool_info =
-		{ FT_TOKEN_WORD, 0, 0, 0, 0, ' ', 0};
 
 	ut_ad(len >= 0);
 
@@ -4636,11 +4632,9 @@ fts_tokenize_document_internal(
 			&str);
 
 		if (str.f_len > 0) {
-			/* JAN: TODO: MySQL 5.7
 			bool_info.position =
 				static_cast<int>(i + inc - str.f_len);
 			ut_ad(bool_info.position >= 0);
-			*/
 
 			/* Stop when add word fails */
 			if (param->mysql_add_word(
@@ -4683,11 +4677,8 @@ fts_tokenize_add_word_for_parser(
 	str.f_n_char = fts_get_token_size(
 		const_cast<CHARSET_INFO*>(param->cs), word, word_len);
 
-	/* JAN: TODO: MySQL 5.7 FTS
 	ut_ad(boolean_info->position >= 0);
 	position = boolean_info->position + fts_param->add_pos;
-	*/
-	position = fts_param->add_pos;
 
 	fts_add_token(result_doc, str, position);
 
diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h
index 84368374430..5bd3f2cd172 100644
--- a/storage/innobase/include/row0ftsort.h
+++ b/storage/innobase/include/row0ftsort.h
@@ -98,6 +98,8 @@ struct fts_psort_t {
 /** Row fts token for plugin parser */
 struct row_fts_token_t {
 	fts_string_t*	text;		/*!< token */
+	ulint		position;	/*!< token position in the
+document */
 	UT_LIST_NODE_T(row_fts_token_t)
 			token_list;	/*!< next token link */
 };
diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index 29e126290cf..c30854ba5fd 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -411,9 +411,7 @@ row_merge_fts_doc_add_word_for_parser(
 	str.f_n_char = fts_get_token_size(
 		(CHARSET_INFO*)param->cs, word, word_len);
 
-	/* JAN: TODO: MySQL 5.7 FTS
 	ut_ad(boolean_info->position >= 0);
-	*/
 
 	ptr = static_cast<byte*>(ut_malloc_nokey(sizeof(row_fts_token_t)
 			+ sizeof(fts_string_t) + str.f_len));
@@ -427,10 +425,7 @@ row_merge_fts_doc_add_word_for_parser(
 	fts_token->text->f_n_char = str.f_n_char;
 	memcpy(fts_token->text->f_str, str.f_str, str.f_len);
 
-	/* JAN: TODO: MySQL 5.7 FTS
 	fts_token->position = boolean_info->position;
-	*/
-
 	/* Add token to list */
 	UT_LIST_ADD_LAST(t_ctx->fts_token_list, fts_token);
 
@@ -667,6 +662,8 @@ row_merge_fts_doc_tokenize(
 		byte		position[4];
 		if (parser == NULL) {
 			pos += t_ctx->processed_len + inc - str.f_len;
+		} else {
+			pos += fts_token->position + t_ctx->init_pos;
 		}
 		len = 4;
 		mach_write_to_4(position, pos);