Bug#31397 Inconsistent drop table behavior of handler tables.

The problem is that DROP TABLE and other DDL statements failed to
automatically close handlers associated with tables that were marked
for reopen (FLUSH TABLES).

The current implementation fails to properly discard handlers of
dropped tables (that were marked for reopen) because it searches
on the open handler tables list and using the current alias of the
table being dropped. The problem is that it must not use the open
handler tables list to search because the table might have been
closed (marked for reopen) by a flush tables command and also it
must not use the current table alias at all since multiple different
aliases may be associated with a single table. This is specially
visible when a user has two open handlers (using alias) of a same
table and a flush tables command is issued before the table is
dropped (see test case). Scanning the handler table list is also
useless for dropping handlers associated with temporary tables,
because temporary tables are not kept in the THD::handler_tables
list.

The solution is to simple scan the handlers hash table searching
for, and deleting all handlers with matching table names if the
reopen flag is not passed to the flush function, indicating that
the handlers should be deleted. All matching handlers are deleted
even if the associated the table is not open.


mysql-test/include/handler.inc:
  Add test case for Bug#31397
mysql-test/r/handler_innodb.result:
  Add test case result for Bug#31397
mysql-test/r/handler_myisam.result:
  Add test case result for Bug#31397
sql/mysql_priv.h:
  Rename flush functions to better match the intent of the caller and
  update functions prototypes and remove unused flags.
sql/sql_base.cc:
  Rename flush functions to better match the intent of the caller.
sql/sql_class.cc:
  Rename the flush functions to better match the intent of the caller.
  The hash_free function is moved to the cleanup.
sql/sql_handler.cc:
  When dropping tables for a final close, scan the handler's hash table since
  the table might not be in the handlers open table list because the table was
  marked for reopen or because it's a temporary table.
sql/sql_rename.cc:
  Drop handlers associated with tables that are being renamed.
sql/sql_table.cc:
  Now that temporary tables are properly removed even when opened
  by a SQL HANDLER, enable the assert since this branch can't be taken
  outside of SF/trigger/prelocked mode.

6 files changed, 123 insertions, 163 deletions

diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h
index 12838317646..f19e11357cd 100644
--- a/sql/mysql_priv.h
+++ b/sql/mysql_priv.h
@@ -1289,12 +1289,9 @@ bool mysql_ha_open(THD *thd, TABLE_LIST *tables, bool reopen);
 bool mysql_ha_close(THD *thd, TABLE_LIST *tables);
 bool mysql_ha_read(THD *, TABLE_LIST *,enum enum_ha_read_modes,char *,
                    List<Item> *,enum ha_rkey_function,Item *,ha_rows,ha_rows);
-int mysql_ha_flush(THD *thd, TABLE_LIST *tables, uint mode_flags,
-                   bool is_locked);
-/* mysql_ha_flush mode_flags bits */
-#define MYSQL_HA_CLOSE_FINAL        0x00
-#define MYSQL_HA_REOPEN_ON_USAGE    0x01
-#define MYSQL_HA_FLUSH_ALL          0x02
+void mysql_ha_flush(THD *thd);
+void mysql_ha_rm_tables(THD *thd, TABLE_LIST *tables);
+void mysql_ha_cleanup(THD *thd);
 
 /* sql_base.cc */
 #define TMP_TABLE_KEY_EXTRA 8
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 66d0cda2155..5e2b0285a1c 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -929,8 +929,8 @@ bool close_cached_tables(THD *thd, bool if_wait_for_refresh,
     thd->proc_info="Flushing tables";
 
     close_old_data_files(thd,thd->open_tables,1,1);
-    mysql_ha_flush(thd, tables, MYSQL_HA_REOPEN_ON_USAGE | MYSQL_HA_FLUSH_ALL,
-                   TRUE);
+    mysql_ha_flush(thd);
+
     bool found=1;
     /* Wait until all threads has closed all the tables we had locked */
     DBUG_PRINT("info",
@@ -2516,7 +2516,7 @@ TABLE *open_table(THD *thd, TABLE_LIST *table_list, MEM_ROOT *mem_root,
     deadlock may occur.
   */
   if (thd->handler_tables)
-    mysql_ha_flush(thd, (TABLE_LIST*) NULL, MYSQL_HA_REOPEN_ON_USAGE, TRUE);
+    mysql_ha_flush(thd);
 
   /*
     Actually try to find the table in the open_cache.
@@ -3136,7 +3136,7 @@ bool wait_for_tables(THD *thd)
   {
     thd->some_tables_deleted=0;
     close_old_data_files(thd,thd->open_tables,0,dropping_tables != 0);
-    mysql_ha_flush(thd, (TABLE_LIST*) NULL, MYSQL_HA_REOPEN_ON_USAGE, TRUE);
+    mysql_ha_flush(thd);
     if (!table_is_used(thd->open_tables,1))
       break;
     (void) pthread_cond_wait(&COND_refresh,&LOCK_open);
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index a904023cbff..a8bac731b87 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -678,9 +678,7 @@ void THD::cleanup(void)
     lock=locked_tables; locked_tables=0;
     close_thread_tables(this);
   }
-  mysql_ha_flush(this, (TABLE_LIST*) 0,
-                 MYSQL_HA_CLOSE_FINAL | MYSQL_HA_FLUSH_ALL, FALSE);
-  hash_free(&handler_tables_hash);
+  mysql_ha_cleanup(this);
   delete_dynamic(&user_var_events);
   hash_free(&user_vars);
   close_temporary_tables(this);
diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc
index 19a99f9d12b..31d6b28a73c 100644
--- a/sql/sql_handler.cc
+++ b/sql/sql_handler.cc
@@ -65,9 +65,6 @@
 static enum enum_ha_read_modes rkey_to_rnext[]=
 { RNEXT_SAME, RNEXT, RPREV, RNEXT, RPREV, RNEXT, RPREV, RPREV };
 
-static int mysql_ha_flush_table(THD *thd, TABLE **table_ptr, uint mode_flags);
-
-
 /*
   Get hash key and hash key length.
 
@@ -119,13 +116,15 @@ static void mysql_ha_hash_free(TABLE_LIST *tables)
 
   @param thd Thread identifier.
   @param tables A list of tables with the first entry to close.
+  @param is_locked If LOCK_open is locked.
 
   @note Though this function takes a list of tables, only the first list entry
   will be closed.
-  @note Broadcasts refresh if it closed the table.
+  @note Broadcasts refresh if it closed a table with old version.
 */
 
-static void mysql_ha_close_table(THD *thd, TABLE_LIST *tables)
+static void mysql_ha_close_table(THD *thd, TABLE_LIST *tables,
+                                 bool is_locked)
 {
   TABLE **table_ptr;
 
@@ -143,13 +142,15 @@ static void mysql_ha_close_table(THD *thd, TABLE_LIST *tables)
   if (*table_ptr)
   {
     (*table_ptr)->file->ha_index_or_rnd_end();
-    VOID(pthread_mutex_lock(&LOCK_open));
+    if (! is_locked)
+      VOID(pthread_mutex_lock(&LOCK_open));
     if (close_thread_table(thd, table_ptr))
     {
       /* Tell threads waiting for refresh that something has happened */
       broadcast_refresh();
     }
-    VOID(pthread_mutex_unlock(&LOCK_open));
+    if (! is_locked)
+      VOID(pthread_mutex_unlock(&LOCK_open));
   }
   else if (tables->table)
   {
@@ -305,7 +306,7 @@ err:
   if (hash_tables)
     my_free((char*) hash_tables, MYF(0));
   if (tables->table)
-    mysql_ha_close_table(thd, tables);
+    mysql_ha_close_table(thd, tables, FALSE);
   DBUG_PRINT("exit",("ERROR"));
   DBUG_RETURN(TRUE);
 }
@@ -339,7 +340,7 @@ bool mysql_ha_close(THD *thd, TABLE_LIST *tables)
                                               (uchar*) tables->alias,
                                               strlen(tables->alias) + 1)))
   {
-    mysql_ha_close_table(thd, hash_tables);
+    mysql_ha_close_table(thd, hash_tables, FALSE);
     hash_delete(&thd->handler_tables_hash, (uchar*) hash_tables);
   }
   else
@@ -478,7 +479,7 @@ retry:
 
   if (need_reopen)
   {
-    mysql_ha_close_table(thd, tables);
+    mysql_ha_close_table(thd, tables, FALSE);
     hash_tables->table= NULL;
     /*
       The lock might have been aborted, we need to manually reset
@@ -669,163 +670,131 @@ err0:
 }
 
 
-/*
-  Flush (close) a list of HANDLER tables.
-
-  SYNOPSIS
-    mysql_ha_flush()
-    thd                         Thread identifier.
-    tables                      The list of tables to close. If NULL,
-                                close all HANDLER tables [marked as flushed].
-    mode_flags                  MYSQL_HA_CLOSE_FINAL finally close the table.
-                                MYSQL_HA_REOPEN_ON_USAGE mark for reopen.
-                                MYSQL_HA_FLUSH_ALL flush all tables, not only
-                                those marked for flush.
-    is_locked                   If LOCK_open is locked.
-
-  DESCRIPTION
-    The list of HANDLER tables may be NULL, in which case all HANDLER
-    tables are closed (if MYSQL_HA_FLUSH_ALL) is set.
-    If 'tables' is NULL and MYSQL_HA_FLUSH_ALL is not set,
-    all HANDLER tables marked for flush are closed.
-    Broadcasts refresh for every table closed.
+/**
+  Scan the handler tables hash for matching tables.
 
-  NOTE
-    Since mysql_ha_flush() is called when the base table has to be closed,
-    we compare real table names, not aliases. Hence, database names matter.
+  @param thd Thread identifier.
+  @param tables The list of tables to remove.
 
-  RETURN
-    0  ok
+  @return Pointer to head of linked list (TABLE_LIST::next_local) of matching
+          TABLE_LIST elements from handler_tables_hash. Otherwise, NULL if no
+          table was matched.
 */
 
-int mysql_ha_flush(THD *thd, TABLE_LIST *tables, uint mode_flags,
-                   bool is_locked)
+static TABLE_LIST *mysql_ha_find(THD *thd, TABLE_LIST *tables)
 {
-  TABLE_LIST    *tmp_tables;
-  TABLE         **table_ptr;
-  bool          did_lock= FALSE;
-  DBUG_ENTER("mysql_ha_flush");
-  DBUG_PRINT("enter", ("tables: 0x%lx  mode_flags: 0x%02x",
-                       (long) tables, mode_flags));
+  TABLE_LIST *hash_tables, *head= NULL, *first= tables;
+  DBUG_ENTER("mysql_ha_find");
 
-  if (tables)
+  /* search for all handlers with matching table names */
+  for (uint i= 0; i < thd->handler_tables_hash.records; i++)
   {
-    /* Close all tables in the list. */
-    for (tmp_tables= tables ; tmp_tables; tmp_tables= tmp_tables->next_local)
+    hash_tables= (TABLE_LIST*) hash_element(&thd->handler_tables_hash, i);
+    for (tables= first; tables; tables= tables->next_local)
     {
-      DBUG_PRINT("info-in-tables-list",("'%s'.'%s' as '%s'",
-                                        tmp_tables->db, tmp_tables->table_name,
-                                        tmp_tables->alias));
-      /* Close all currently open handler tables with the same base table. */
-      table_ptr= &(thd->handler_tables);
-      while (*table_ptr)
-      {
-        if ((!*tmp_tables->db ||
-             !my_strcasecmp(&my_charset_latin1, (*table_ptr)->s->db.str,
-                             tmp_tables->db)) &&
-            ! my_strcasecmp(&my_charset_latin1,
-                            (*table_ptr)->s->table_name.str,
-                            tmp_tables->table_name))
-        {
-          DBUG_PRINT("info",("*table_ptr '%s'.'%s' as '%s'",
-                             (*table_ptr)->s->db.str,
-                             (*table_ptr)->s->table_name.str,
-                             (*table_ptr)->alias));
-          /* The first time it is required, lock for close_thread_table(). */
-          if (! did_lock && ! is_locked)
-          {
-            VOID(pthread_mutex_lock(&LOCK_open));
-            did_lock= TRUE;
-          }
-          mysql_ha_flush_table(thd, table_ptr, mode_flags);
-          continue;
-        }
-        table_ptr= &(*table_ptr)->next;
-      }
-      /* end of handler_tables list */
+      if ((! *tables->db ||
+          ! my_strcasecmp(&my_charset_latin1, hash_tables->db, tables->db)) &&
+          ! my_strcasecmp(&my_charset_latin1, hash_tables->table_name,
+                          tables->table_name))
+        break;
     }
-    /* end of flush tables list */
-  }
-  else
-  {
-    /* Close all currently open tables [which are marked for flush]. */
-    table_ptr= &(thd->handler_tables);
-    while (*table_ptr)
+    if (tables)
     {
-      if ((mode_flags & MYSQL_HA_FLUSH_ALL) ||
-          (*table_ptr)->needs_reopen_or_name_lock())
-      {
-        /* The first time it is required, lock for close_thread_table(). */
-        if (! did_lock && ! is_locked)
-        {
-          VOID(pthread_mutex_lock(&LOCK_open));
-          did_lock= TRUE;
-        }
-        mysql_ha_flush_table(thd, table_ptr, mode_flags);
-        continue;
-      }
-      table_ptr= &(*table_ptr)->next;
+      hash_tables->next_local= head;
+      head= hash_tables;
     }
   }
 
-  /* Release the lock if it was taken by this function. */
-  if (did_lock)
-    VOID(pthread_mutex_unlock(&LOCK_open));
+  DBUG_RETURN(head);
+}
+
+
+/**
+  Remove matching tables from the HANDLER's hash table.
+
+  @param thd Thread identifier.
+  @param tables The list of tables to remove.
+
+  @note Broadcasts refresh if it closed a table with old version.
+*/
+
+void mysql_ha_rm_tables(THD *thd, TABLE_LIST *tables)
+{
+  TABLE_LIST *hash_tables, *next;
+  DBUG_ENTER("mysql_ha_rm_tables");
+
+  safe_mutex_assert_not_owner(&LOCK_open);
+
+  DBUG_ASSERT(tables);
 
-  DBUG_RETURN(0);
+  hash_tables= mysql_ha_find(thd, tables);
+
+  while (hash_tables)
+  {
+    next= hash_tables->next_local;
+    if (hash_tables->table)
+      mysql_ha_close_table(thd, hash_tables, FALSE);
+    hash_delete(&thd->handler_tables_hash, (uchar*) hash_tables);
+    hash_tables= next;
+  }
+
+  DBUG_VOID_RETURN;
 }
 
-/*
-  Flush (close) a table.
 
-  SYNOPSIS
-    mysql_ha_flush_table()
-    thd                         Thread identifier.
-    table                       The table to close.
-    mode_flags                  MYSQL_HA_CLOSE_FINAL finally close the table.
-                                MYSQL_HA_REOPEN_ON_USAGE mark for reopen.
+/**
+  Flush (close and mark for re-open) all tables that should be should
+  be reopen.
 
-  DESCRIPTION
-    Broadcasts refresh if it closed the table.
-    The caller must lock LOCK_open.
+  @param thd Thread identifier.
 
-  RETURN
-    0  ok
+  @note Broadcasts refresh if it closed a table with old version.
 */
 
-static int mysql_ha_flush_table(THD *thd, TABLE **table_ptr, uint mode_flags)
+void mysql_ha_flush(THD *thd)
 {
-  TABLE_LIST    *hash_tables;
-  TABLE         *table= *table_ptr;
-  DBUG_ENTER("mysql_ha_flush_table");
-  DBUG_PRINT("enter",("'%s'.'%s' as '%s'  flags: 0x%02x",
-                      table->s->db.str, table->s->table_name.str,
-                      table->alias, mode_flags));
+  TABLE_LIST *hash_tables;
+  DBUG_ENTER("mysql_ha_flush");
 
-  if ((hash_tables= (TABLE_LIST*) hash_search(&thd->handler_tables_hash,
-                                              (uchar*) table->alias,
-                                              strlen(table->alias) + 1)))
+  safe_mutex_assert_owner(&LOCK_open);
+
+  for (uint i= 0; i < thd->handler_tables_hash.records; i++)
   {
-    if (! (mode_flags & MYSQL_HA_REOPEN_ON_USAGE))
-    {
-      /* This is a final close. Remove from hash. */
-      hash_delete(&thd->handler_tables_hash, (uchar*) hash_tables);
-    }
-    else
+    hash_tables= (TABLE_LIST*) hash_element(&thd->handler_tables_hash, i);
+    if (hash_tables->table && hash_tables->table->needs_reopen_or_name_lock())
     {
+      mysql_ha_close_table(thd, hash_tables, TRUE);
       /* Mark table as closed, ready for re-open. */
       hash_tables->table= NULL;
     }
-  }    
+  }
 
-  safe_mutex_assert_owner(&LOCK_open);
-  (*table_ptr)->file->ha_index_or_rnd_end();
-  safe_mutex_assert_owner(&LOCK_open);
-  if (close_thread_table(thd, table_ptr))
+  DBUG_VOID_RETURN;
+}
+
+
+/**
+  Close all HANDLER's tables.
+
+  @param thd Thread identifier.
+
+  @note Broadcasts refresh if it closed a table with old version.
+*/
+
+void mysql_ha_cleanup(THD *thd)
+{
+  TABLE_LIST *hash_tables;
+  DBUG_ENTER("mysql_ha_cleanup");
+
+  for (uint i= 0; i < thd->handler_tables_hash.records; i++)
   {
-    /* Tell threads waiting for refresh that something has happened */
-    broadcast_refresh();
-  }
+    hash_tables= (TABLE_LIST*) hash_element(&thd->handler_tables_hash, i);
+    if (hash_tables->table)
+      mysql_ha_close_table(thd, hash_tables, FALSE);
+   }
 
-  DBUG_RETURN(0);
+  hash_free(&thd->handler_tables_hash);
+
+  DBUG_VOID_RETURN;
 }
+
diff --git a/sql/sql_rename.cc b/sql/sql_rename.cc
index 750bcd50479..66d89edc146 100644
--- a/sql/sql_rename.cc
+++ b/sql/sql_rename.cc
@@ -51,6 +51,8 @@ bool mysql_rename_tables(THD *thd, TABLE_LIST *table_list, bool silent)
     DBUG_RETURN(1);
   }
 
+  mysql_ha_rm_tables(thd, table_list);
+
   if (wait_if_global_read_lock(thd,0,1))
     DBUG_RETURN(1);
 
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index bf4a02ae5cc..9d26273e809 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -1521,6 +1521,8 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
       built_query.append("DROP TABLE ");
   }
 
+  mysql_ha_rm_tables(thd, tables);
+
   pthread_mutex_lock(&LOCK_open);
 
   /*
@@ -1562,8 +1564,6 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
     handlerton *table_type;
     enum legacy_db_type frm_db_type;
 
-    mysql_ha_flush(thd, table, MYSQL_HA_CLOSE_FINAL, 1);
-
     error= drop_temporary_table(thd, table);
 
     switch (error) {
@@ -1572,13 +1572,7 @@ int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
       tmp_table_deleted= 1;
       continue;
     case -1:
-      // table already in use
-      /*
-        XXX: This branch should never be taken outside of SF, trigger or
-             prelocked mode.
-
-        DBUG_ASSERT(thd->in_sub_stmt);
-      */
+      DBUG_ASSERT(thd->in_sub_stmt);
       error= 1;
       goto err_with_placeholders;
     default:
@@ -4025,7 +4019,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
                             Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF))
     DBUG_RETURN(TRUE);
 
-  mysql_ha_flush(thd, tables, MYSQL_HA_CLOSE_FINAL, FALSE);
+  mysql_ha_rm_tables(thd, tables);
+
   for (table= tables; table; table= table->next_local)
   {
     char table_name[NAME_LEN*2+2];
@@ -5766,8 +5761,7 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
   build_table_filename(reg_path, sizeof(reg_path), db, table_name, reg_ext, 0);
   build_table_filename(path, sizeof(path), db, table_name, "", 0);
 
-
-  mysql_ha_flush(thd, table_list, MYSQL_HA_CLOSE_FINAL, FALSE);
+  mysql_ha_rm_tables(thd, table_list);
 
   /* DISCARD/IMPORT TABLESPACE is always alone in an ALTER TABLE */
   if (alter_info->tablespace_op != NO_TABLESPACE_OP)