MDEV-9847: Window functions: crash with big_tables=1

- Move filesort's sort_positions argument into class Filesort.
- Make window function code construct Filesort with sort_positions=true.

9 files changed, 60 insertions, 22 deletions

diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result
index ca13c32e1f4..eb41f4faea5 100644
--- a/mysql-test/r/win.result
+++ b/mysql-test/r/win.result
@@ -1694,3 +1694,24 @@ EXPLAIN
   }
 }
 drop table t1;
+#
+# MDEV-9847: Window functions: crash with big_tables=1
+#
+create table t1(a int);
+insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+set @tmp=@@big_tables;
+set big_tables=1;
+select rank() over (order by a) from t1;
+rank() over (order by a)
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+set big_tables=@tmp;
+drop table t1;
diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test
index b13722f8f26..f03e3773c03 100644
--- a/mysql-test/t/win.test
+++ b/mysql-test/t/win.test
@@ -1063,3 +1063,15 @@ from t1;
 
 drop table t1;
 
+
+--echo #
+--echo # MDEV-9847: Window functions: crash with big_tables=1
+--echo #
+create table t1(a int);
+insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
+set @tmp=@@big_tables;
+set big_tables=1;
+select rank() over (order by a) from t1;
+set big_tables=@tmp;
+drop table t1;
+
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 78b74380f7e..82a5c90d0bb 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -124,17 +124,12 @@ void Sort_param::init_for_filesort(uint sortlen, TABLE *table,
   @param      thd            Current thread
   @param      table          Table to sort
   @param      filesort       How to sort the table
-  @param      sort_positions Set to TRUE if we want to force sorting by
-			     position
-                             (Needed by UPDATE/INSERT or ALTER TABLE or
-                              when rowids are required by executor)
-                             applying WHERE condition.
   @param[out] found_rows     Store the number of found rows here.
                              This is the number of found rows after
                              applying WHERE condition.
   @note
-    If we sort by position (like if sort_positions is 1) filesort() will
-    call table->prepare_for_position().
+    If we sort by position (like if filesort->sort_positions==true) 
+    filesort() will call table->prepare_for_position().
 
   @retval
     0			Error
@@ -142,7 +137,6 @@ void Sort_param::init_for_filesort(uint sortlen, TABLE *table,
 */
 
 SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort,
-                    bool sort_positions,
                     Filesort_tracker* tracker)
 {
   int error;
@@ -203,7 +197,7 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort,
                                      &multi_byte_charset),
                           table,
                           thd->variables.max_length_for_sort_data,
-                          max_rows, sort_positions);
+                          max_rows, filesort->sort_positions);
 
   sort->addon_buf=    param.addon_buf;
   sort->addon_field=  param.addon_field;
diff --git a/sql/filesort.h b/sql/filesort.h
index 6d665dbe0aa..18029a10c14 100644
--- a/sql/filesort.h
+++ b/sql/filesort.h
@@ -47,16 +47,25 @@ public:
   bool own_select;
   /** true means we are using Priority Queue for order by with limit. */
   bool using_pq;
+  
+  /* 
+    TRUE means sort operation must produce table rowids. 
+    FALSE means that it halso has an option of producing {sort_key,
+    addon_fields} pairs.
+  */
+  bool sort_positions;
 
   Filesort_tracker *tracker;
 
-  Filesort(ORDER *order_arg, ha_rows limit_arg, SQL_SELECT *select_arg):
+  Filesort(ORDER *order_arg, ha_rows limit_arg, bool sort_positions_arg,
+           SQL_SELECT *select_arg):
     order(order_arg),
     limit(limit_arg),
     sortorder(NULL),
     select(select_arg),
     own_select(false), 
-    using_pq(false)
+    using_pq(false),
+    sort_positions(sort_positions_arg)
   {
     DBUG_ASSERT(order);
   };
@@ -143,12 +152,10 @@ public:
   { return filesort_buffer.sort_buffer_size(); }
 
   friend SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort,
-                             bool sort_positions,
                              Filesort_tracker* tracker);
 };
 
 SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort,
-                    bool sort_positions,
                     Filesort_tracker* tracker);
 
 void change_double_for_sort(double nr,uchar *to);
diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index 0a76ee0a699..95aee805c7f 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -492,12 +492,13 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
   {
 
     {
-      Filesort fsort(order, HA_POS_ERROR, select);
+      Filesort fsort(order, HA_POS_ERROR, true, select);
       DBUG_ASSERT(query_plan.index == MAX_KEY);
+
       Filesort_tracker *fs_tracker= 
         thd->lex->explain->get_upd_del_plan()->filesort_tracker;
 
-      if (!(file_sort= filesort(thd, table, &fsort, true, fs_tracker)))
+      if (!(file_sort= filesort(thd, table, &fsort, fs_tracker)))
         goto got_error;
 
       thd->inc_examined_row_count(file_sort->examined_rows);
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 7c2d2c53592..44e00b099ed 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -2814,7 +2814,9 @@ JOIN::optimize_distinct()
 bool
 JOIN::add_sorting_to_table(JOIN_TAB *tab, ORDER *order)
 {
-  tab->filesort= new (thd->mem_root) Filesort(order, HA_POS_ERROR, tab->select);
+  tab->filesort= 
+    new (thd->mem_root) Filesort(order, HA_POS_ERROR, tab->keep_current_rowid,
+                                 tab->select);
   if (!tab->filesort)
     return true;
   /*
@@ -21279,7 +21281,7 @@ create_sort_index(THD *thd, JOIN *join, JOIN_TAB *tab, Filesort *fsort)
 
   if (table->s->tmp_table)
     table->file->info(HA_STATUS_VARIABLE);	// Get record count
-  file_sort= filesort(thd, table, fsort, tab->keep_current_rowid, fsort->tracker);
+  file_sort= filesort(thd, table, fsort, fsort->tracker);
   DBUG_ASSERT(tab->filesort_result == 0);
   tab->filesort_result= file_sort;
   tab->records= 0;
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 583058f80a5..9542b6b54fe 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -9447,13 +9447,14 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to,
 
       THD_STAGE_INFO(thd, stage_sorting);
       Filesort_tracker dummy_tracker(false);
-      Filesort fsort(order, HA_POS_ERROR, NULL);
+      Filesort fsort(order, HA_POS_ERROR, true, NULL);
+
       if (thd->lex->select_lex.setup_ref_array(thd, order_num) ||
           setup_order(thd, thd->lex->select_lex.ref_pointer_array,
                       &tables, fields, all_fields, order))
         goto err;
 
-      if (!(file_sort= filesort(thd, from, &fsort, true, &dummy_tracker)))
+      if (!(file_sort= filesort(thd, from, &fsort, &dummy_tracker)))
         goto err;
     }
     thd_progress_next_stage(thd);
diff --git a/sql/sql_update.cc b/sql/sql_update.cc
index 739bef82ab2..6c60350844e 100644
--- a/sql/sql_update.cc
+++ b/sql/sql_update.cc
@@ -558,12 +558,12 @@ int mysql_update(THD *thd,
 	to update
         NOTE: filesort will call table->prepare_for_position()
       */
-      Filesort fsort(order, limit, select);
+      Filesort fsort(order, limit, true, select);
 
       Filesort_tracker *fs_tracker= 
         thd->lex->explain->get_upd_del_plan()->filesort_tracker;
 
-      if (!(file_sort= filesort(thd, table, &fsort, true, fs_tracker)))
+      if (!(file_sort= filesort(thd, table, &fsort, fs_tracker)))
 	goto err;
       thd->inc_examined_row_count(file_sort->examined_rows);
 
diff --git a/sql/sql_window.cc b/sql/sql_window.cc
index 1a405ec198e..e479faa73d3 100644
--- a/sql/sql_window.cc
+++ b/sql/sql_window.cc
@@ -1834,7 +1834,7 @@ bool Window_funcs_sort::setup(THD *thd, SQL_SELECT *sel,
   ORDER* sort_order= concat_order_lists(thd->mem_root, 
                                         spec->partition_list->first,
                                         spec->order_list->first);
-  filesort= new (thd->mem_root) Filesort(sort_order, HA_POS_ERROR, NULL);
+  filesort= new (thd->mem_root) Filesort(sort_order, HA_POS_ERROR, true, NULL);
   /* Apply the same condition that the subsequent sort has. */
   filesort->select= sel;