Bug #11880012: INDEX_SUBQUERY, BLACKHOLE,

               HANG IN PREPARING WITH 100% CPU USAGE

Infinite loop in the subselect_indexsubquery_engine::exec()
function caused Server hang with 100% CPU usage.

The BLACKHOLE storage engine didn't update handler's
table->status variable after index operations, that
caused an infinite "while(!table->status)" execution.

Index access methods of the BLACKHOLE engine handler
have been updated to set table->status variable to
STATUS_NOT_FOUND or 0 when such a method returns a
HA_ERR_END_OF_FILE error or 0 respectively.


mysql-test/r/blackhole.result:
  Bug #11880012: INDEX_SUBQUERY, BLACKHOLE,
                 HANG IN PREPARING WITH 100% CPU USAGE
  
  New test file for the BLACKHOLE engine.
mysql-test/t/blackhole.test:
  Bug #11880012: INDEX_SUBQUERY, BLACKHOLE,
                 HANG IN PREPARING WITH 100% CPU USAGE
  
  New test file for the BLACKHOLE engine.
storage/blackhole/ha_blackhole.cc:
  Bug #11880012: INDEX_SUBQUERY, BLACKHOLE,
                 HANG IN PREPARING WITH 100% CPU USAGE
  
  Index access methods of the BLACKHOLE engine handler
  have been updated to set table->status variable to
  STATUS_NOT_FOUND or 0 when such a method returns a
  HA_ERR_END_OF_FILE error or 0 respectively, like
  we do in MyISAM storage engine.

3 files changed, 40 insertions, 1 deletions

diff --git a/mysql-test/r/blackhole.result b/mysql-test/r/blackhole.result
new file mode 100644
index 00000000000..317070ca08e
--- /dev/null
+++ b/mysql-test/r/blackhole.result
@@ -0,0 +1,11 @@
+#
+# Bug #11880012: INDEX_SUBQUERY, BLACKHOLE,
+#                HANG IN PREPARING WITH 100% CPU USAGE
+#
+CREATE TABLE t1(a INT NOT NULL);
+INSERT INTO t1 VALUES (1), (2), (3);
+CREATE TABLE t2 (a INT UNSIGNED, b INT, UNIQUE KEY (a, b)) ENGINE=BLACKHOLE;
+SELECT 1 FROM t1 WHERE a = ANY (SELECT a FROM t2);
+1
+DROP TABLE t1, t2;
+End of 5.5 tests 
diff --git a/mysql-test/t/blackhole.test b/mysql-test/t/blackhole.test
new file mode 100644
index 00000000000..1451f7606e0
--- /dev/null
+++ b/mysql-test/t/blackhole.test
@@ -0,0 +1,21 @@
+#
+# Tests for the BLACKHOLE storage engine
+#
+
+--source include/have_blackhole.inc
+
+--echo #
+--echo # Bug #11880012: INDEX_SUBQUERY, BLACKHOLE,
+--echo #                HANG IN PREPARING WITH 100% CPU USAGE
+--echo #
+
+CREATE TABLE t1(a INT NOT NULL);
+INSERT INTO t1 VALUES (1), (2), (3);
+CREATE TABLE t2 (a INT UNSIGNED, b INT, UNIQUE KEY (a, b)) ENGINE=BLACKHOLE;
+
+SELECT 1 FROM t1 WHERE a = ANY (SELECT a FROM t2);
+
+DROP TABLE t1, t2;
+
+--echo End of 5.5 tests 
+
diff --git a/storage/blackhole/ha_blackhole.cc b/storage/blackhole/ha_blackhole.cc
index c12d5afbfa5..9d0f89915a5 100644
--- a/storage/blackhole/ha_blackhole.cc
+++ b/storage/blackhole/ha_blackhole.cc
@@ -151,6 +151,7 @@ int ha_blackhole::rnd_next(uchar *buf)
   else
     rc= HA_ERR_END_OF_FILE;
   MYSQL_READ_ROW_DONE(rc);
+  table->status= rc ? STATUS_NOT_FOUND : 0;
   DBUG_RETURN(rc);
 }
 
@@ -241,6 +242,7 @@ int ha_blackhole::index_read_map(uchar * buf, const uchar * key,
   else
     rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= rc ? STATUS_NOT_FOUND : 0;
   DBUG_RETURN(rc);
 }
 
@@ -258,6 +260,7 @@ int ha_blackhole::index_read_idx_map(uchar * buf, uint idx, const uchar * key,
   else
     rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= rc ? STATUS_NOT_FOUND : 0;
   DBUG_RETURN(rc);
 }
 
@@ -274,6 +277,7 @@ int ha_blackhole::index_read_last_map(uchar * buf, const uchar * key,
   else
     rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= rc ? STATUS_NOT_FOUND : 0;
   DBUG_RETURN(rc);
 }
 
@@ -285,6 +289,7 @@ int ha_blackhole::index_next(uchar * buf)
   MYSQL_INDEX_READ_ROW_START(table_share->db.str, table_share->table_name.str);
   rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= STATUS_NOT_FOUND;
   DBUG_RETURN(rc);
 }
 
@@ -296,6 +301,7 @@ int ha_blackhole::index_prev(uchar * buf)
   MYSQL_INDEX_READ_ROW_START(table_share->db.str, table_share->table_name.str);
   rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= STATUS_NOT_FOUND;
   DBUG_RETURN(rc);
 }
 
@@ -307,8 +313,8 @@ int ha_blackhole::index_first(uchar * buf)
   MYSQL_INDEX_READ_ROW_START(table_share->db.str, table_share->table_name.str);
   rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= STATUS_NOT_FOUND;
   DBUG_RETURN(rc);
-  DBUG_RETURN(HA_ERR_END_OF_FILE);
 }
 
 
@@ -319,6 +325,7 @@ int ha_blackhole::index_last(uchar * buf)
   MYSQL_INDEX_READ_ROW_START(table_share->db.str, table_share->table_name.str);
   rc= HA_ERR_END_OF_FILE;
   MYSQL_INDEX_READ_ROW_DONE(rc);
+  table->status= STATUS_NOT_FOUND;
   DBUG_RETURN(rc);
 }