fixed convertion and handling IN subqueries with rows (BUG#11867)

mysql-test/r/subselect.result:
  testst of IN subqueries with row
mysql-test/t/subselect.test:
  tests of ion subqueries with row
sql/item.h:
  add method to prevent of removing Item_ref_null_helper from HAVING
sql/item_cmpfunc.h:
  Prevented removing of Item_test_isnotnull from HAVING
sql/item_subselect.cc:
  fixed converting row IN subqueries
sql/sql_select.cc:
  fixed debug print

4 files changed, 151 insertions, 37 deletions

diff --git a/sql/item.h b/sql/item.h
index 825b37fe64c..c71769b5129 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -1023,6 +1023,15 @@ public:
   String* val_str(String* s);
   bool get_date(TIME *ltime, uint fuzzydate);
   void print(String *str);
+  /*
+    we add RAND_TABLE_BIT to prevent moving this item from HAVING to WHERE
+  */
+  table_map used_tables() const
+  {
+    return (depended_from ?
+            OUTER_REF_TABLE_BIT :
+            (*ref)->used_tables() | RAND_TABLE_BIT);
+  }
 };
 
 class Item_null_helper :public Item_ref_null_helper
diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index 525f269e528..47884f6064e 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -832,6 +832,11 @@ public:
   longlong val_int();
   const char *func_name() const { return "<is_not_null_test>"; }
   void update_used_tables();
+  /*
+    we add RAND_TABLE_BIT to prevent moving this item from HAVING to WHERE
+  */
+  table_map used_tables() const
+    { return used_tables_cache | RAND_TABLE_BIT; }
 };
 
 
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 6ec8b11d863..c8405a9f8f4 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -859,6 +859,7 @@ Item_in_subselect::single_value_transformer(JOIN *join,
 	argument (reference) to fix_fields()
       */
       select_lex->where= join->conds= and_items(join->conds, item);
+      select_lex->where->top_level_item();
       if (join->conds->fix_fields(thd, join->tables_list, 0))
 	DBUG_RETURN(RES_ERROR);
     }
@@ -912,10 +913,13 @@ Item_in_subselect::single_value_transformer(JOIN *join,
 Item_subselect::trans_res
 Item_in_subselect::row_value_transformer(JOIN *join)
 {
-  DBUG_ENTER("Item_in_subselect::row_value_transformer");
-
-  Item *item= 0;
   SELECT_LEX *select_lex= join->select_lex;
+  Item *having_item= 0;
+  uint cols_num= left_expr->cols();
+  bool is_having_used= (join->having || select_lex->with_sum_func ||
+                        select_lex->group_list.first ||
+                        !select_lex->table_list.elements);
+  DBUG_ENTER("Item_in_subselect::row_value_transformer");
 
   if (select_lex->item_list.elements != left_expr->cols())
   {
@@ -946,61 +950,156 @@ Item_in_subselect::row_value_transformer(JOIN *join)
   }
 
   select_lex->uncacheable|= UNCACHEABLE_DEPENDENT;
+  if (is_having_used)
   {
-    uint n= left_expr->cols();
-    List_iterator_fast<Item> li(select_lex->item_list);
-    for (uint i= 0; i < n; i++)
+    /*
+      (l1, l2, l3) IN (SELECT v1, v2, v3 ... HAVING having) =>
+      EXISTS (SELECT ... HAVING having and
+                                (l1 = v1 or is null v1) and
+                                (l2 = v2 or is null v2) and
+                                (l3 = v3 or is null v3) and
+                                is_not_null_test(v1) and
+                                is_not_null_test(v2) and
+                                is_not_null_test(v3))
+      where is_not_null_test used to register nulls in case if we have
+      not found matching to return correct NULL value
+    */
+    Item *item_having_part2= 0;
+    for (uint i= 0; i < cols_num; i++)
     {
-      Item *func;
       DBUG_ASSERT(left_expr->fixed && select_lex->ref_pointer_array[i]->fixed);
       if (select_lex->ref_pointer_array[i]->
           check_cols(left_expr->el(i)->cols()))
         DBUG_RETURN(RES_ERROR);
-      if (join->having || select_lex->with_sum_func ||
-          select_lex->group_list.elements)
-        func= new Item_ref_null_helper(this,
-				       select_lex->ref_pointer_array+i,
-				       (char *) "<no matter>",
-				       (char *) "<list ref>");
-      else
-        func= li++;
-      func=
-	eq_creator.create(new Item_direct_ref((*optimizer->get_cache())->
-					      addr(i),
-					      (char *)"<no matter>",
-					      (char *)in_left_expr_name),
-			  func);
-      item= and_items(item, func);
+      Item *item_eq=
+        new Item_func_eq(new
+                         Item_direct_ref((*optimizer->get_cache())->
+                                         addr(i),
+                                         (char *)"<no matter>",
+                                         (char *)in_left_expr_name),
+                         new
+                         Item_direct_ref(select_lex->ref_pointer_array + i,
+                                         (char *)"<no matter>",
+                                         (char *)"<list ref>")
+                        );
+      Item *item_isnull=
+        new Item_func_isnull(new
+                             Item_direct_ref( select_lex->
+                                              ref_pointer_array+i,
+                                              (char *)"<no matter>",
+                                              (char *)"<list ref>")
+                            );
+      having_item=
+        and_items(having_item,
+                  new Item_cond_or(item_eq, item_isnull));
+      item_having_part2=
+        and_items(item_having_part2,
+                  new
+                  Item_is_not_null_test(this,
+                                        new
+                                        Item_direct_ref(select_lex->
+                                                        ref_pointer_array + i,
+                                                        (char *)"<no matter>",
+                                                        (char *)"<list ref>")
+                                       )
+                 );
+      item_having_part2->top_level_item();
     }
+    having_item= and_items(having_item, item_having_part2);
+    having_item->top_level_item();
   }
-  if (join->having || select_lex->with_sum_func ||
-      select_lex->group_list.first ||
-      !select_lex->table_list.elements)
+  else
   {
     /*
+      (l1, l2, l3) IN (SELECT v1, v2, v3 ... WHERE where) =>
+      EXISTS (SELECT ... WHERE where and
+                               (l1 = v1 or is null v1) and
+                               (l2 = v2 or is null v2) and
+                               (l3 = v3 or is null v3)
+                         HAVING is_not_null_test(v1) and
+                                is_not_null_test(v2) and
+                                is_not_null_test(v3))
+      where is_not_null_test register NULLs values but reject rows
+
+      in case when we do not need correct NULL, we have simplier construction:
+      EXISTS (SELECT ... WHERE where and
+                               (l1 = v1) and
+                               (l2 = v2) and
+                               (l3 = v3)
+    */
+    Item *where_item= 0;
+    for (uint i= 0; i < cols_num; i++)
+    {
+      Item *item, *item_isnull;
+      DBUG_ASSERT(left_expr->fixed && select_lex->ref_pointer_array[i]->fixed);
+      if (select_lex->ref_pointer_array[i]->
+          check_cols(left_expr->el(i)->cols()))
+        DBUG_RETURN(RES_ERROR);
+      item=
+        new Item_func_eq(new
+                         Item_direct_ref((*optimizer->get_cache())->
+                                         addr(i),
+                                         (char *)"<no matter>",
+                                         (char *)in_left_expr_name),
+                         new
+                         Item_direct_ref( select_lex->
+                                          ref_pointer_array+i,
+                                          (char *)"<no matter>",
+                                          (char *)"<list ref>")
+                        );
+      if (!abort_on_null)
+      {
+        having_item=
+          and_items(having_item,
+                    new
+                    Item_is_not_null_test(this,
+                                          new
+                                          Item_direct_ref(select_lex->
+                                                          ref_pointer_array + i,
+                                                          (char *)"<no matter>",
+                                                          (char *)"<list ref>")
+                                         )
+                  );
+        item_isnull= new
+          Item_func_isnull(new
+                           Item_direct_ref( select_lex->
+                                            ref_pointer_array+i,
+                                            (char *)"<no matter>",
+                                            (char *)"<list ref>")
+                          );
+
+        item= new Item_cond_or(item, item_isnull);
+      }
+
+      where_item= and_items(where_item, item);
+    }
+    /*
       AND can't be changed during fix_fields()
-      we can assign select_lex->having here, and pass 0 as last
+      we can assign select_lex->where here, and pass 0 as last
       argument (reference) to fix_fields()
     */
-    select_lex->having= join->having= and_items(join->having, item);
-    select_lex->having_fix_field= 1;
-    if (join->having->fix_fields(thd, join->tables_list, 0))
-    {
-      select_lex->having_fix_field= 0;
+    select_lex->where= join->conds= and_items(join->conds, where_item);
+    select_lex->where->top_level_item();
+    if (join->conds->fix_fields(thd, join->tables_list, 0))
       DBUG_RETURN(RES_ERROR);
-    }
-    select_lex->having_fix_field= 0;
   }
-  else
+  if (having_item)
   {
+    bool res;
+    select_lex->having= join->having= and_items(join->having, having_item);
+    select_lex->having->top_level_item();
     /*
       AND can't be changed during fix_fields()
       we can assign select_lex->having here, and pass 0 as last
       argument (reference) to fix_fields()
     */
-    select_lex->where= join->conds= and_items(join->conds, item);
-    if (join->conds->fix_fields(thd, join->tables_list, 0))
+    select_lex->having_fix_field= 1;
+    res= join->having->fix_fields(thd, join->tables_list, 0);
+    select_lex->having_fix_field= 0;
+    if (res)
+    {
       DBUG_RETURN(RES_ERROR);
+    }
   }
   DBUG_RETURN(RES_OK);
 }
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 353ccf79ab8..a7984b525d1 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1439,7 +1439,8 @@ JOIN::exec()
 	curr_join->tmp_having= make_cond_for_table(curr_join->tmp_having,
 						   ~ (table_map) 0,
 						   ~used_tables);
-	DBUG_EXECUTE("where",print_where(conds,"having after sort"););
+	DBUG_EXECUTE("where",print_where(curr_join->tmp_having,
+                                         "having after sort"););
       }
     }
     {