MDEV-20770 Server crashes in JOIN::transform_in_predicates_into_in_subq upon 2nd execution of PS/SP comparing GEOMETRY with other types.bb-10.3-mdev-20770-hf

The Item_in_subselect::in_strategy keeps the value and as the error
happens the condition isn't modified. That leads to wrong ::fix_fields
execution on second PS run. Also the select->table_list is merged
but not restored if an error happens, which causes hanging loops on
the third PS execution.

3 files changed, 65 insertions, 19 deletions

diff --git a/mysql-test/main/subselect_sj.result b/mysql-test/main/subselect_sj.result
index af602c26060..13b8a16080f 100644
--- a/mysql-test/main/subselect_sj.result
+++ b/mysql-test/main/subselect_sj.result
@@ -3309,4 +3309,22 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	<subquery2>	eq_ref	distinct_key	distinct_key	4	func	1	
 2	MATERIALIZED	t4	ALL	NULL	NULL	NULL	NULL	500	
 drop table t1, t2, t3, t4;
+# 
+# MDEV-20770: Server crashes in JOIN::transform_in_predicates_into_in_subq
+#   upon 2nd execution of PS/SP comparing GEOMETRY with other types
+# 
+CREATE TABLE t1 (a GEOMETRY);
+CREATE TABLE t2 (b INT);
+INSERT INTO t1 VALUES (GeomFromText('POINT(0 0)')),(GeomFromText('POINT(1 1)'));
+INSERT INTO t2 VALUES (1),(2);
+PREPARE stmt FROM "SELECT * from t1 WHERE a IN (SELECT b FROM t2)";
+EXECUTE stmt;
+ERROR HY000: Illegal parameter data types geometry and int for operation '='
+EXECUTE stmt;
+ERROR HY000: Illegal parameter data types geometry and int for operation '='
+EXECUTE stmt;
+ERROR HY000: Illegal parameter data types geometry and int for operation '='
+EXECUTE stmt;
+ERROR HY000: Illegal parameter data types geometry and int for operation '='
+DROP TABLE t1, t2;
 set optimizer_switch=@subselect_sj_tmp;
diff --git a/mysql-test/main/subselect_sj.test b/mysql-test/main/subselect_sj.test
index 1ca10cf4090..236116d0018 100644
--- a/mysql-test/main/subselect_sj.test
+++ b/mysql-test/main/subselect_sj.test
@@ -2993,5 +2993,28 @@ explain select * from t3 where a in (select a from t4);
 
 drop table t1, t2, t3, t4;
 
+--echo # 
+--echo # MDEV-20770: Server crashes in JOIN::transform_in_predicates_into_in_subq
+--echo #   upon 2nd execution of PS/SP comparing GEOMETRY with other types
+--echo # 
+
+CREATE TABLE t1 (a GEOMETRY);
+CREATE TABLE t2 (b INT);
+ 
+INSERT INTO t1 VALUES (GeomFromText('POINT(0 0)')),(GeomFromText('POINT(1 1)'));
+INSERT INTO t2 VALUES (1),(2);
+
+PREPARE stmt FROM "SELECT * from t1 WHERE a IN (SELECT b FROM t2)";
+--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION
+EXECUTE stmt;
+--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION
+EXECUTE stmt;
+--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION
+EXECUTE stmt;
+--error ER_ILLEGAL_PARAMETER_DATA_TYPES2_FOR_OPERATION
+EXECUTE stmt;
+ 
+DROP TABLE t1, t2;
+
 # The following command must be the last one the file 
 set optimizer_switch=@subselect_sj_tmp;
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index d65e00f4b97..6080717e5f5 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -1531,6 +1531,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
 {
   SELECT_LEX *parent_lex= parent_join->select_lex;
   TABLE_LIST *emb_tbl_nest= NULL;
+  TABLE_LIST *orig_tl;
   List<TABLE_LIST> *emb_join_list= &parent_lex->top_join_list;
   THD *thd= parent_join->thd;
   DBUG_ENTER("convert_subq_to_sj");
@@ -1692,17 +1693,17 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
      because view's tables are inserted after the view)
   */
   
-  for (tl= (TABLE_LIST*)(parent_lex->table_list.first); tl->next_local; tl= tl->next_local)
+  for (orig_tl= (TABLE_LIST*)(parent_lex->table_list.first);
+       orig_tl->next_local;
+       orig_tl= orig_tl->next_local)
   {}
 
-  tl->next_local= subq_lex->join->tables_list;
+  orig_tl->next_local= subq_lex->join->tables_list;
 
   /* A theory: no need to re-connect the next_global chain */
 
   /* 3. Remove the original subquery predicate from the WHERE/ON */
 
-  // The subqueries were replaced for Item_int(1) earlier
-  subq_pred->reset_strategy(SUBS_SEMI_JOIN);       // for subsequent executions
   /*TODO: also reset the 'm_with_subquery' there. */
 
   /* n. Adjust the parent_join->table_count counter */
@@ -1735,12 +1736,14 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     Add the subquery-induced equalities too.
   */
   SELECT_LEX *save_lex= thd->lex->current_select;
+  table_map subq_pred_used_tables;
+
   thd->lex->current_select=subq_lex;
   if (subq_pred->left_expr->fix_fields_if_needed(thd, &subq_pred->left_expr))
-    DBUG_RETURN(TRUE);
+    goto restore_tl_and_exit;
   thd->lex->current_select=save_lex;
 
-  table_map subq_pred_used_tables= subq_pred->used_tables();
+  subq_pred_used_tables= subq_pred->used_tables();
   sj_nest->nested_join->sj_corr_tables= subq_pred_used_tables;
   sj_nest->nested_join->sj_depends_on=  subq_pred_used_tables |
                                         subq_pred->left_expr->used_tables();
@@ -1783,7 +1786,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
       new (thd->mem_root) Item_func_eq(thd, subq_pred->left_expr_orig,
                                        subq_lex->ref_pointer_array[0]);
     if (!item_eq)
-      DBUG_RETURN(TRUE);
+      goto restore_tl_and_exit;
     if (subq_pred->left_expr_orig != subq_pred->left_expr)
       thd->change_item_tree(item_eq->arguments(), subq_pred->left_expr);
     item_eq->in_equality_no= 0;
@@ -1804,7 +1807,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
         Item_func_eq(thd, subq_pred->left_expr_orig->element_index(i),
                      subq_lex->ref_pointer_array[i]);
       if (!item_eq)
-        DBUG_RETURN(TRUE);
+        goto restore_tl_and_exit;
       DBUG_ASSERT(subq_pred->left_expr->element_index(i)->fixed);
       if (subq_pred->left_expr_orig->element_index(i) !=
           subq_pred->left_expr->element_index(i))
@@ -1823,13 +1826,13 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     Item_row *row= new (thd->mem_root) Item_row(thd, subq_lex->pre_fix);
     /* fix fields on subquery was call so they should be the same */
     if (!row)
-      DBUG_RETURN(TRUE);
+      goto restore_tl_and_exit;
     DBUG_ASSERT(subq_pred->left_expr->cols() == row->cols());
     nested_join->sj_outer_expr_list.push_back(&subq_pred->left_expr);
     Item_func_eq *item_eq=
       new (thd->mem_root) Item_func_eq(thd, subq_pred->left_expr_orig, row);
     if (!item_eq)
-      DBUG_RETURN(TRUE);
+      goto restore_tl_and_exit;
     for (uint i= 0; i < row->cols(); i++)
     {
       if (row->element_index(i) != subq_lex->ref_pointer_array[i])
@@ -1848,9 +1851,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     we have in here).
   */
   if (sj_nest->sj_on_expr->fix_fields_if_needed(thd, &sj_nest->sj_on_expr))
-  {
-    DBUG_RETURN(TRUE);
-  }
+    goto restore_tl_and_exit;
 
   /*
     Walk through sj nest's WHERE and ON expressions and call
@@ -1875,9 +1876,7 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     emb_tbl_nest->on_expr->top_level_item();
     if (emb_tbl_nest->on_expr->fix_fields_if_needed(thd,
                                                     &emb_tbl_nest->on_expr))
-    {
-      DBUG_RETURN(TRUE);
-    }
+      goto restore_tl_and_exit;
   }
   else
   {
@@ -1891,9 +1890,8 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     save_lex= thd->lex->current_select;
     thd->lex->current_select=parent_join->select_lex;
     if (parent_join->conds->fix_fields_if_needed(thd, &parent_join->conds))
-    {
-      DBUG_RETURN(1);
-    }
+      goto restore_tl_and_exit;
+
     thd->lex->current_select=save_lex;
     parent_join->select_lex->where= parent_join->conds;
   }
@@ -1906,9 +1904,16 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
       parent_lex->ftfunc_list->push_front(ifm, thd->mem_root);
   }
 
+  // The subqueries were replaced for Item_int(1) earlier
+  subq_pred->reset_strategy(SUBS_SEMI_JOIN);       // for subsequent executions
+
   parent_lex->have_merged_subqueries= TRUE;
   /* Fatal error may have been set to by fix_after_pullout() */
   DBUG_RETURN(thd->is_fatal_error);
+
+restore_tl_and_exit:
+  orig_tl->next_local= NULL;
+  DBUG_RETURN(TRUE);
 }