MDEV-8208: Sporadic SEGFAULT on startup

Problem:
When mysqld starts as a galera node, it creates 2 system threads
(applier & rollbacker) using start_wsrep_THD(). These threads are
created before plugin initialization (plugin_init()) for SST methods
like rsync and xtrabackup.

The threads' initialization itself can proceed in parallel to mysqld's
main thread of execution. As a result, the thread initialization code
(start_wsrep_THD()) can end up accessing some un/partially initialized
structures (like maria_hton, in this particular case) resulting in
segfault.

Solution:
Fixed by calling THD::init_for_queries() (which accesses maria_hton)
only after the plugins have been initialized.

1 files changed, 31 insertions, 12 deletions

diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index d72d9fdf151..947574e37aa 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -91,6 +91,12 @@ my_bool wsrep_slave_UK_checks          = 0; // slave thread does UK checks
 my_bool wsrep_slave_FK_checks          = 0; // slave thread does FK checks
 bool wsrep_new_cluster                 = false; // Bootstrap the cluster ?
 
+/*
+  Set during the creation of first wsrep applier and rollback threads.
+  Since these threads are critical, abort if the thread creation fails.
+*/
+my_bool wsrep_creating_startup_threads = 0;
+
 // Use wsrep_gtid_domain_id for galera transactions?
 bool wsrep_gtid_mode                   = 0;
 // gtid_domain_id for galera transactions.
@@ -792,6 +798,7 @@ void wsrep_init_startup (bool first)
 
   if (!wsrep_start_replication()) unireg_abort(1);
 
+  wsrep_creating_startup_threads= 1;
   wsrep_create_rollbacker();
   wsrep_create_appliers(1);
 
@@ -1719,16 +1726,11 @@ pthread_handler_t start_wsrep_THD(void *arg)
   THD *thd;
   wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg;
 
-  if (my_thread_init())
+  if (my_thread_init() || (!(thd= new THD(true))))
   {
-    WSREP_ERROR("Could not initialize thread");
-    return(NULL);
+    goto error;
   }
 
-  if (!(thd= new THD(true)))
-  {
-    return(NULL);
-  }
   mysql_mutex_lock(&LOCK_thread_count);
   thd->thread_id=thread_id++;
 
@@ -1765,7 +1767,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
     statistic_increment(aborted_connects,&LOCK_status);
     MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
 
-    return(NULL);
+    goto error;
   }
 
 // </5.1.17>
@@ -1788,8 +1790,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
     statistic_increment(aborted_connects,&LOCK_status);
     MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
     delete thd;
-
-    return(NULL);
+    goto error;
   }
 
   thd->system_thread= SYSTEM_THREAD_SLAVE_SQL;
@@ -1799,12 +1800,21 @@ pthread_handler_t start_wsrep_THD(void *arg)
   //thd->version= refresh_version;
   thd->proc_info= 0;
   thd->set_command(COM_SLEEP);
-  thd->set_time();
-  thd->init_for_queries();
+
+  if (plugins_are_initialized)
+  {
+    thd->init_for_queries();
+  }
 
   mysql_mutex_lock(&LOCK_thread_count);
   wsrep_running_threads++;
   mysql_cond_broadcast(&COND_thread_count);
+
+  if (wsrep_running_threads > 2)
+  {
+    wsrep_creating_startup_threads= 0;
+  }
+
   mysql_mutex_unlock(&LOCK_thread_count);
 
   processor(thd);
@@ -1842,6 +1852,15 @@ pthread_handler_t start_wsrep_THD(void *arg)
     mysql_mutex_unlock(&LOCK_thread_count);
   }
   return(NULL);
+
+error:
+  WSREP_ERROR("Failed to create/initialize system thread");
+
+  /* Abort if its the first applier/rollbacker thread. */
+  if (wsrep_creating_startup_threads < 2)
+    unireg_abort(1);
+  else
+    return NULL;
 }