MDEV-10232: Scalar result of subquery changes after adding an outer select stmt

In a subquery, we don't have to maintain order
Added a fix such that order is considered when there is a limit clause.

3 files changed, 25 insertions, 2 deletions

diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result
index 8b912e1bfe9..3783ba12db2 100644
--- a/mysql-test/r/subselect4.result
+++ b/mysql-test/r/subselect4.result
@@ -2442,3 +2442,15 @@ i
 drop table t1, t2, t3;
 SET optimizer_switch= @@global.optimizer_switch;
 set @@tmp_table_size= @@global.tmp_table_size;
+#
+# MDEV-10232 Scalar result of subquery changes after adding an outer select stmt
+#
+create table t1(c1 int, c2 int, primary key(c2));
+insert into t1 values(2,1),(1,2);
+select (select c1 from t1 group by c1,c2 order by c1 limit 1) as x;
+x
+1
+(select c1 from t1 group by c1,c2 order by c1 limit 1);
+c1
+1
+drop table t1;
diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test
index cb102f8319e..253160c46ac 100644
--- a/mysql-test/t/subselect4.test
+++ b/mysql-test/t/subselect4.test
@@ -1999,3 +1999,13 @@ drop table t1, t2, t3;
 
 SET optimizer_switch= @@global.optimizer_switch;
 set @@tmp_table_size= @@global.tmp_table_size;
+
+--echo #
+--echo # MDEV-10232 Scalar result of subquery changes after adding an outer select stmt
+--echo #
+
+create table t1(c1 int, c2 int, primary key(c2));
+insert into t1 values(2,1),(1,2);
+select (select c1 from t1 group by c1,c2 order by c1 limit 1) as x;
+(select c1 from t1 group by c1,c2 order by c1 limit 1);
+drop table t1;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 1bd101bbe6f..e37c1b8c280 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1619,7 +1619,8 @@ JOIN::optimize_inner()
         <fields> to ORDER BY <fields>. There are three exceptions:
         - if skip_sort_order is set (see above), then we can simply skip
           GROUP BY;
-        - if we are in a subquery, we don't have to maintain order
+        - if we are in a subquery, we don't have to maintain order unless there
+	  is a limit clause in the subquery.
         - we can only rewrite ORDER BY if the ORDER BY fields are 'compatible'
           with the GROUP BY ones, i.e. either one is a prefix of another.
           We only check if the ORDER BY is a prefix of GROUP BY. In this case
@@ -1631,7 +1632,7 @@ JOIN::optimize_inner()
       if (!order || test_if_subpart(group_list, order))
       {
         if (skip_sort_order ||
-            select_lex->master_unit()->item) // This is a subquery
+            (select_lex->master_unit()->item && select_limit == HA_POS_ERROR)) // This is a subquery
           order= NULL;
         else
           order= group_list;