MDEV-3899 Valgrind warnings (blocks are definitely lost) in filesort on IN subquery with SUM and DISTINCT

  
Analysys:
In the beginning of JOIN::cleanup there is code that is supposed to
free all filesort buffers. The code assumes that the table being sorted
is the first non-constant table. To get this table it calls:
first_top_level_tab(this, WITHOUT_CONST_TABLES)
  
However, first_top_level_tab() instead returned the wrong table - the first
one in the plan, instead of the first non-constant table. There is no other
place outside filesort() where sort buffers may be freed. As a result, the
sort buffer was not freed, and there was a memory leak.
  
Solution:
Change first_top_level_tab(), to test for WITH_CONST_TABLES instead of
WITHOUT_CONST_TABLES.

3 files changed, 30 insertions, 2 deletions

diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result
index c35075d80a1..4fda55a2619 100644
--- a/mysql-test/r/subselect4.result
+++ b/mysql-test/r/subselect4.result
@@ -2247,5 +2247,18 @@ MAX(a)	bb
 NULL	NULL
 drop table t1, t2;
 set optimizer_switch=@subselect4_tmp;
+#
+# MDEV-3899  Valgrind warnings (blocks are definitely lost) in filesort on IN subquery with SUM and DISTINCT
+#
+CREATE TABLE t1 (a INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(9);
+CREATE TABLE t2 (b INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (8);
+SELECT * FROM t1 
+WHERE (1, 1) IN (SELECT a, SUM(DISTINCT a) FROM t1, t2 GROUP BY a);
+a
+1
+9
+drop table t1, t2;
 SET optimizer_switch= @@global.optimizer_switch;
 set @@tmp_table_size= @@global.tmp_table_size;
diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test
index 50b34eece89..b22be893a78 100644
--- a/mysql-test/t/subselect4.test
+++ b/mysql-test/t/subselect4.test
@@ -1777,5 +1777,20 @@ drop table t1, t2;
 
 set optimizer_switch=@subselect4_tmp;
 
+--echo #
+--echo # MDEV-3899  Valgrind warnings (blocks are definitely lost) in filesort on IN subquery with SUM and DISTINCT
+--echo #
+
+CREATE TABLE t1 (a INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1),(9);
+
+CREATE TABLE t2 (b INT) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (8);
+
+SELECT * FROM t1 
+WHERE (1, 1) IN (SELECT a, SUM(DISTINCT a) FROM t1, t2 GROUP BY a);
+
+drop table t1, t2;
+
 SET optimizer_switch= @@global.optimizer_switch;
 set @@tmp_table_size= @@global.tmp_table_size;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 8f7fdab4ed3..8914341e0f3 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -7333,10 +7333,10 @@ JOIN_TAB *next_breadth_first_tab(JOIN *join, enum enum_exec_or_opt tabs_kind,
 }
 
 
-JOIN_TAB *first_top_level_tab(JOIN *join, enum enum_with_const_tables with_const)
+JOIN_TAB *first_top_level_tab(JOIN *join, enum enum_with_const_tables const_tbls)
 {
   JOIN_TAB *tab= join->join_tab;
-  if (with_const == WITH_CONST_TABLES)
+  if (const_tbls == WITHOUT_CONST_TABLES)
   {
     if (join->const_tables == join->table_count)
       return NULL;