Bug#37627: Killing query with sum(exists()) or avg(exists()) reproducibly crashes server

When there is an error executing EXISTS predicates they return NULL as their string
or decimal value but don't set the NULL value flag.
Fixed by returning 0 (as a decimal or a string) on error exectuting the subquery.
Note that we can't return NULL as EXISTS is not supposed to return NULL.

mysql-test/r/subselect.result:
  Bug#37627: test case
mysql-test/t/subselect.test:
  Bug#37627: test case
sql/item_subselect.cc:
  Bug#37627: return decimal (or string) 0 isntead of a NULL pointer on
  error calculating an EXISTS predicate.

3 files changed, 61 insertions, 6 deletions

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index 923bc2254bd..04983ef19ab 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -4398,3 +4398,16 @@ INSERT INTO t1 VALUES (1), (3);
 SELECT * FROM t2 WHERE b NOT IN (SELECT max(t.c) FROM t1, t1 t WHERE t.c>10);
 a	b
 DROP TABLE t1,t2;
+CREATE TABLE t1(id INT);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d;
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c;
+SET SESSION debug="d,subselect_exec_fail";
+SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
+SUM(EXISTS(SELECT RAND() FROM t1))
+0
+SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
+REVERSE(EXISTS(SELECT RAND() FROM t1))
+0
+SET SESSION debug=DEFAULT;
+DROP TABLE t1;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 85b21c83612..08e0f6592ab 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -3298,3 +3298,19 @@ INSERT INTO t1 VALUES (1), (3);
 SELECT * FROM t2 WHERE b NOT IN (SELECT max(t.c) FROM t1, t1 t WHERE t.c>10);
 
 DROP TABLE t1,t2;
+
+#
+# Bug #37627: Killing query with sum(exists()) or avg(exists()) reproducibly
+# crashes server
+#
+
+CREATE TABLE t1(id INT);
+INSERT INTO t1 VALUES (1),(2),(3),(4);
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c,t1 d;
+INSERT INTO t1 SELECT a.id FROM t1 a,t1 b,t1 c;
+# Setup the mysqld to crash at certain point
+SET SESSION debug="d,subselect_exec_fail";
+SELECT SUM(EXISTS(SELECT RAND() FROM t1)) FROM t1;
+SELECT REVERSE(EXISTS(SELECT RAND() FROM t1));
+SET SESSION debug=DEFAULT;
+DROP TABLE t1;
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index ea16f3c3518..3981b91a27c 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -254,6 +254,11 @@ bool Item_subselect::exec()
   if (thd->is_error())
   /* Do not execute subselect in case of a fatal error */
     return 1;
+  /*
+    Simulate a failure in sub-query execution. Used to test e.g.
+    out of memory or query being killed conditions.
+  */
+  DBUG_EXECUTE_IF("subselect_exec_fail", return 1;);
 
   res= engine->exec();
 
@@ -719,27 +724,48 @@ longlong Item_exists_subselect::val_int()
   return value;
 }
 
+
+/**
+  Return the result of EXISTS as a string value
+
+  Converts the true/false result into a string value.
+  Note that currently this cannot be NULL, so if the query exection fails
+  it will return 0.
+
+  @param decimal_value[out]    buffer to hold the resulting string value
+  @retval                      Pointer to the converted string.
+                               Can't be a NULL pointer, as currently
+                               EXISTS cannot return NULL.
+*/
+
 String *Item_exists_subselect::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   if (exec())
-  {
     reset();
-    return 0;
-  }
   str->set((ulonglong)value,&my_charset_bin);
   return str;
 }
 
 
+/**
+  Return the result of EXISTS as a decimal value
+
+  Converts the true/false result into a decimal value.
+  Note that currently this cannot be NULL, so if the query exection fails
+  it will return 0.
+
+  @param decimal_value[out]    Buffer to hold the resulting decimal value
+  @retval                      Pointer to the converted decimal.
+                               Can't be a NULL pointer, as currently
+                               EXISTS cannot return NULL.
+*/
+
 my_decimal *Item_exists_subselect::val_decimal(my_decimal *decimal_value)
 {
   DBUG_ASSERT(fixed == 1);
   if (exec())
-  {
     reset();
-    return 0;
-  }
   int2my_decimal(E_DEC_FATAL_ERROR, value, 0, decimal_value);
   return decimal_value;
 }