MDEV-294 SELECT WHERE ST_CONTAINS doesn't return all the records where ST_CONTAINS() is 1.

        Optimizator fails using index with ST_Within(g, constant_poly).

per-file comments:
  mysql-test/r/gis-rt-precise.result
        test result fixed.
  mysql-test/r/gis-rtree.result
        test result fixed.
  mysql-test/suite/maria/r/maria-gis-rtree-dynamic.result
        test result fixed.
  mysql-test/suite/maria/r/maria-gis-rtree-trans.result
        test result fixed.
  mysql-test/suite/maria/r/maria-gis-rtree.result
        test result fixed.
  storage/maria/ma_rt_index.c
        Use MBR_INTERSECT mode when optimizing the select WITH ST_Within.
  storage/myisam/rt_index.c
        Use MBR_INTERSECT mode when optimizing the select WITH ST_Within.

1 files changed, 22 insertions, 4 deletions

diff --git a/storage/maria/ma_rt_index.c b/storage/maria/ma_rt_index.c
index f4c5b26f40a..e0f8a5dca41 100644
--- a/storage/maria/ma_rt_index.c
+++ b/storage/maria/ma_rt_index.c
@@ -196,6 +196,12 @@ int maria_rtree_find_first(MARIA_HA *info, MARIA_KEY *key, uint32 search_flag)
   uint nod_cmp_flag;
   MARIA_KEYDEF *keyinfo= key->keyinfo;
 
+  /*
+    At the moment index can only properly handle the
+    MBR_INTERSECT, so we use it for all sorts of queries.
+    TODO: better searsh for CONTAINS/WITHIN.
+  */
+  search_flag= nod_cmp_flag= MBR_INTERSECT;
   if ((root= info->s->state.key_root[keyinfo->key_nr]) == HA_OFFSET_ERROR)
   {
     my_errno= HA_ERR_END_OF_FILE;
@@ -213,8 +219,11 @@ int maria_rtree_find_first(MARIA_HA *info, MARIA_KEY *key, uint32 search_flag)
   info->maria_rtree_recursion_depth= -1;
   info->keyread_buff_used= 1;
 
-  nod_cmp_flag= ((search_flag & (MBR_EQUAL | MBR_WITHIN)) ?
-                 MBR_WITHIN : MBR_INTERSECT);
+  /*
+    TODO better search for CONTAINS/WITHIN.
+    nod_cmp_flag= ((search_flag & (MBR_EQUAL | MBR_WITHIN)) ?
+                   MBR_WITHIN : MBR_INTERSECT);
+  */
   return maria_rtree_find_req(info, keyinfo, search_flag, nod_cmp_flag, root,
                               0);
 }
@@ -241,6 +250,12 @@ int maria_rtree_find_next(MARIA_HA *info, uint keynr, uint32 search_flag)
   uint32 nod_cmp_flag;
   MARIA_KEYDEF *keyinfo= info->s->keyinfo + keynr;
   DBUG_ASSERT(info->last_key.keyinfo == keyinfo);
+  /*
+    At the moment index can only properly handle the
+    MBR_INTERSECT, so we use it for all sorts of queries.
+    TODO: better searsh for CONTAINS/WITHIN.
+  */
+  search_flag= nod_cmp_flag= MBR_INTERSECT;
 
   if (info->update & HA_STATE_DELETED)
     return maria_rtree_find_first(info, &info->last_key, search_flag);
@@ -284,8 +299,11 @@ int maria_rtree_find_next(MARIA_HA *info, uint keynr, uint32 search_flag)
     return -1;
   }
 
-  nod_cmp_flag= (((search_flag & (MBR_EQUAL | MBR_WITHIN)) ?
-                  MBR_WITHIN : MBR_INTERSECT));
+  /*
+    TODO better search for CONTAINS/WITHIN.
+    nod_cmp_flag= (((search_flag & (MBR_EQUAL | MBR_WITHIN)) ?
+                    MBR_WITHIN : MBR_INTERSECT));
+  */
   return maria_rtree_find_req(info, keyinfo, search_flag, nod_cmp_flag, root,
                               0);
 }