BUG#29445 - match ... against () never returns

Part 2:
Searching fulltext index for a word with boolean mode truncation
operator may cause infinite loop.

The problem was that "smarter index merge" was used with "trunc-words",
which must never happen.

Affects 5.1 only.


mysql-test/r/fulltext.result:
  Addition to a test case for BUG#29445.
mysql-test/t/fulltext.test:
  Addition to a test case for BUG#29445.
storage/myisam/ft_boolean_search.c:
  Fulltext "smarter index merge" optimization assumes that rows
  it gets are ordered by doc_id. That is not the case when we
  search for a word with truncation operator. It may return
  rows in random order. Thus we may not use "smarter index merge"
  optimization with "trunc-words".
  
  Also fixed compiler warning introduced by Part 1 patch.

3 files changed, 14 insertions, 4 deletions

diff --git a/mysql-test/r/fulltext.result b/mysql-test/r/fulltext.result
index 02b115cb6dc..a649f70b6f2 100644
--- a/mysql-test/r/fulltext.result
+++ b/mysql-test/r/fulltext.result
@@ -485,7 +485,10 @@ INSERT INTO t1 VALUES('Offside'),('City Of God');
 SELECT a FROM t1 WHERE MATCH a AGAINST ('+city of*' IN BOOLEAN MODE);
 a
 City Of God
-SELECT a FROM t1 WHERE MATCH a AGAINST ('+city (of)*' IN BOOLEAN MODE);
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city (of*)' IN BOOLEAN MODE);
+a
+City Of God
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city* of*' IN BOOLEAN MODE);
 a
 City Of God
 DROP TABLE t1;
diff --git a/mysql-test/t/fulltext.test b/mysql-test/t/fulltext.test
index b1bf0036c70..ecb4a0e5691 100644
--- a/mysql-test/t/fulltext.test
+++ b/mysql-test/t/fulltext.test
@@ -414,7 +414,8 @@ DROP TABLE t1;
 CREATE TABLE t1(a VARCHAR(20), FULLTEXT(a));
 INSERT INTO t1 VALUES('Offside'),('City Of God');
 SELECT a FROM t1 WHERE MATCH a AGAINST ('+city of*' IN BOOLEAN MODE);
-SELECT a FROM t1 WHERE MATCH a AGAINST ('+city (of)*' IN BOOLEAN MODE);
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city (of*)' IN BOOLEAN MODE);
+SELECT a FROM t1 WHERE MATCH a AGAINST ('+city* of*' IN BOOLEAN MODE);
 DROP TABLE t1;
 
 # End of 4.1 tests
diff --git a/storage/myisam/ft_boolean_search.c b/storage/myisam/ft_boolean_search.c
index 5c421d3688b..c881f7a7480 100644
--- a/storage/myisam/ft_boolean_search.c
+++ b/storage/myisam/ft_boolean_search.c
@@ -23,6 +23,12 @@
   inside plus subtree. max_docid could be used by any word in plus
   subtree, but it could be updated by plus-word only.
 
+  Fulltext "smarter index merge" optimization assumes that rows
+  it gets are ordered by doc_id. That is not the case when we
+  search for a word with truncation operator. It may return
+  rows in random order. Thus we may not use "smarter index merge"
+  optimization with "trunc-words".
+
   The idea is: there is no need to search for docid smaller than
   biggest docid inside current plus subtree or any upper plus subtree.
 
@@ -443,7 +449,7 @@ static int _ft2_search(FTB *ftb, FTB_WORD *ftbw, my_bool init_search)
     memcpy(lastkey_buf+off, info->lastkey, info->lastkey_length);
   }
   ftbw->docid[0]=info->lastpos;
-  if (ftbw->flags & FTB_FLAG_YES)
+  if (ftbw->flags & FTB_FLAG_YES && !(ftbw->flags & FTB_FLAG_TRUNC))
     ftbw->max_docid_expr->max_docid= info->lastpos;
   return 0;
 }
@@ -488,7 +494,7 @@ static void _ftb_init_index_search(FT_INFO *ftb)
       {
         if (ftbe->flags & FTB_FLAG_NO ||                     /* 2 */
             ftbe->up->ythresh - ftbe->up->yweaks >
-            test(ftbe->flags & FTB_FLAG_YES))                /* 1 */
+            (uint) test(ftbe->flags & FTB_FLAG_YES))         /* 1 */
         {
           FTB_EXPR *top_ftbe=ftbe->up;
           ftbw->docid[0]=HA_OFFSET_ERROR;