MDEV-17066: Bytes lost or Assertion `status_var.local_memory_used == 0 after DELETE with subquery with ROLLUP

The issue here is when records are read from the temporary file
(filesort result in this case) via a cache(rr_from_cache).
The cache is initialized with init_rr_cache.
For correlated subquery the cache allocation is happening at each execution
of the subquery but the deallocation happens only once and that was
when the query execution was done.

So generally for subqueries we do two types of cleanup

1) Full cleanup: we should free all resources of the query(like temp tables).
   This is done generally when the query execution is complete or the subquery
   re-execution is not needed (case with uncorrelated subquery)

2) Partial cleanup: Minor cleanup that is required if
   the subquery needs recalculation. This is done for all the structures that
   need to be allocated for each execution (example SORT_INFO for filesort
   is allocated for each execution of the correlated subquery).

The fix here would be free the cache used by rr_from_cache in the partial
cleanup phase.

6 files changed, 99 insertions, 25 deletions

diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result
index a21b4174aae..ad8df9f9836 100644
--- a/mysql-test/r/subselect4.result
+++ b/mysql-test/r/subselect4.result
@@ -2620,4 +2620,29 @@ SELECT (SELECT DISTINCT t1i.b FROM t1 t1i GROUP BY t1i.a ORDER BY MAX(t1o.b)) FR
 0
 SET @@optimizer_switch= @save_optimizer_switch;
 DROP TABLE t1;
+#
+# MDEV-17066: Bytes lost or Assertion `status_var.local_memory_used == 0 after DELETE
+# with subquery with ROLLUP
+#
+CREATE TABLE t1 (i INT DEFAULT 0, c VARCHAR(2048));
+INSERT INTO t1  SELECT 0, seq FROM seq_1_to_6000;
+CREATE TABLE t2 (f VARCHAR(2048) DEFAULT '');
+INSERT INTO t2 VALUES ('1'),('bar');
+EXPLAIN
+SELECT * FROM t2 WHERE f IN ( SELECT MAX(c) FROM t1 GROUP BY c WITH ROLLUP);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DEPENDENT SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	6000	Using filesort
+SELECT * FROM t2 WHERE f IN ( SELECT MAX(c) FROM t1 GROUP BY c WITH ROLLUP);
+f
+1
+SELECT * FROM t2;
+f
+1
+bar
+DELETE FROM t2 WHERE f IN ( SELECT MAX(c) FROM t1 GROUP BY c WITH ROLLUP );
+SELECT * FROM t2;
+f
+bar
+DROP TABLE t1, t2;
 # End of 10.2 tests
diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test
index c33fe15bdcc..f35df5b6967 100644
--- a/mysql-test/t/subselect4.test
+++ b/mysql-test/t/subselect4.test
@@ -7,6 +7,8 @@ drop table if exists t0,t1,t2,t3,t4,t5,t6;
 drop view if exists v1, v2;
 --enable_warnings
 
+--source include/have_sequence.inc
+
 set @subselect4_tmp= @@optimizer_switch;
 set optimizer_switch='semijoin=on,firstmatch=on,loosescan=on';
 set optimizer_switch='semijoin_with_cache=on';
@@ -2153,4 +2155,25 @@ SELECT (SELECT DISTINCT t1i.b FROM t1 t1i GROUP BY t1i.a ORDER BY MAX(t1o.b)) FR
 SET @@optimizer_switch= @save_optimizer_switch;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-17066: Bytes lost or Assertion `status_var.local_memory_used == 0 after DELETE
+--echo # with subquery with ROLLUP
+--echo #
+
+CREATE TABLE t1 (i INT DEFAULT 0, c VARCHAR(2048));
+INSERT INTO t1  SELECT 0, seq FROM seq_1_to_6000;
+
+CREATE TABLE t2 (f VARCHAR(2048) DEFAULT '');
+INSERT INTO t2 VALUES ('1'),('bar');
+
+EXPLAIN
+SELECT * FROM t2 WHERE f IN ( SELECT MAX(c) FROM t1 GROUP BY c WITH ROLLUP);
+SELECT * FROM t2 WHERE f IN ( SELECT MAX(c) FROM t1 GROUP BY c WITH ROLLUP);
+
+SELECT * FROM t2;
+DELETE FROM t2 WHERE f IN ( SELECT MAX(c) FROM t1 GROUP BY c WITH ROLLUP );
+SELECT * FROM t2;
+
+DROP TABLE t1, t2;
+
 --echo # End of 10.2 tests
diff --git a/sql/records.cc b/sql/records.cc
index 6a611d46ca4..8a152cda4c2 100644
--- a/sql/records.cc
+++ b/sql/records.cc
@@ -320,12 +320,9 @@ bool init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
 
 
 void end_read_record(READ_RECORD *info)
-{                   /* free cache if used */
-  if (info->cache)
-  {
-    my_free_lock(info->cache);
-    info->cache=0;
-  }
+{
+  /* free cache if used */
+  free_cache(info);
   if (info->table)
   {
     if (info->table->is_created())
@@ -336,6 +333,17 @@ void end_read_record(READ_RECORD *info)
   }
 }
 
+
+void free_cache(READ_RECORD *info)
+{
+  if (info->cache)
+  {
+    my_free_lock(info->cache);
+    info->cache=0;
+  }
+}
+
+
 static int rr_handle_error(READ_RECORD *info, int error)
 {
   if (info->thd->killed)
diff --git a/sql/records.h b/sql/records.h
index b5f04dbd161..dd63d3608bb 100644
--- a/sql/records.h
+++ b/sql/records.h
@@ -30,6 +30,7 @@ class SORT_INFO;
 struct READ_RECORD;
 
 void end_read_record(READ_RECORD *info);
+void free_cache(READ_RECORD *info);
 
 /**
   A context for reading through a single table using a chosen access method:
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 379a109c57c..96e9602d77d 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -12391,24 +12391,7 @@ void JOIN::cleanup(bool full)
       for (tab= first_linear_tab(this, WITH_BUSH_ROOTS, WITH_CONST_TABLES); tab;
            tab= next_linear_tab(this, tab, WITH_BUSH_ROOTS))
       {
-        if (!tab->table)
-          continue;
-        DBUG_PRINT("info", ("close index: %s.%s  alias: %s",
-                            tab->table->s->db.str,
-                            tab->table->s->table_name.str,
-                            tab->table->alias.c_ptr()));
-	if (tab->table->is_created())
-        {
-          tab->table->file->ha_index_or_rnd_end();
-          if (tab->aggr)
-          {
-            int tmp= 0;
-            if ((tmp= tab->table->file->extra(HA_EXTRA_NO_CACHE)))
-              tab->table->file->print_error(tmp, MYF(0));
-          }
-        }
-        delete tab->filesort_result;
-        tab->filesort_result= NULL;
+        tab->partial_cleanup();
       }
     }
   }
@@ -26982,6 +26965,40 @@ void JOIN::handle_implicit_grouping_with_window_funcs()
   }
 }
 
+
+/*
+  @brief
+    Perform a partial cleanup for the JOIN_TAB structure
+
+  @note
+    this is used to cleanup resources for the re-execution of correlated
+    subqueries.
+*/
+void JOIN_TAB::partial_cleanup()
+{
+  if (!table)
+    return;
+
+  if (table->is_created())
+  {
+    table->file->ha_index_or_rnd_end();
+    DBUG_PRINT("info", ("close index: %s.%s  alias: %s",
+               table->s->db.str,
+               table->s->table_name.str,
+               table->alias.c_ptr()));
+    if (aggr)
+    {
+      int tmp= 0;
+      if ((tmp= table->file->extra(HA_EXTRA_NO_CACHE)))
+        table->file->print_error(tmp, MYF(0));
+    }
+  }
+  delete filesort_result;
+  filesort_result= NULL;
+  free_cache(&read_record);
+}
+
+
 /**
   @} (end of group Query_Optimizer)
 */
diff --git a/sql/sql_select.h b/sql/sql_select.h
index 92da1355be0..4584460ca3f 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -613,7 +613,7 @@ typedef struct st_join_table {
   bool use_order() const; ///< Use ordering provided by chosen index?
   bool sort_table();
   bool remove_duplicates();
-
+  void partial_cleanup();
 } JOIN_TAB;