core: Efficient ap_thread_current() when apr_thread_local() is missing.

#define ap_thread_create, ap_thread_current_create and ap_thread_current to
their apr-1.8+ equivalent if available, or implement them using the compiler's
thread_local mechanism if available, or finally provide stubs otherwise.

#define AP_HAS_THREAD_LOCAL to 1 in the two former case or 0 otherwise, while
AP_THREAD_LOCAL is defined to the compiler's keyword iff AP_HAS_THREAD_LOCAL.

Replace all apr_thread_create() calls with ap_thread_create() so that httpd
threads can use ap_thread_current()'s pool data as Thread Local Storage.

Bump MMN minor.

* include/httpd.h():
  Define AP_HAS_THREAD_LOCAL, AP_THREAD_LOCAL (eventually), ap_thread_create(),
  ap_thread_current_create() and ap_thread_current().
  
* server/util.c:
  Implement ap_thread_create(), ap_thread_current_create() and
  ap_thread_current() when APR < 1.8.

* modules/core/mod_watchdog.c, modules/http2/h2_workers.c,
    modules/ssl/mod_ssl_ct.c:
  Use ap_thread_create() instead of apr_thread_create.

* server/main.c:
  Use AP_HAS_THREAD_LOCAL and ap_thread_current_create instead of APR's.

* server/util_pcre.c:
  Use AP_HAS_THREAD_LOCAL and ap_thread_current instead of APR's.

* server/mpm/event/event.c, server/mpm/worker/worker.c,
    server/mpm/prefork/prefork.c:
  Use ap_thread_create() instead of apr_thread_create.
  Create an apr_thread_t/ap_thread_current() for the main chaild thread usable
  at child_init().
  
* server/mpm/winnt/child.c:
  Use ap_thread_create() instead of CreateThread().
  Create an apr_thread_t/ap_thread_current() for the main chaild thread usable



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1897460 13f79535-47bb-0310-9956-ffa450edef68

12 files changed, 340 insertions, 81 deletions

diff --git a/include/ap_mmn.h b/include/ap_mmn.h
index 25709472db..6e35c4fee1 100644
--- a/include/ap_mmn.h
+++ b/include/ap_mmn.h
@@ -700,7 +700,9 @@
  *                         add PROXY_WORKER_UDS_PATH_SIZE.
  * 20211221.1 (2.5.1-dev)  Add read_line to scoreboard.
  * 20211221.2 (2.5.1-dev)  Add AGAIN, AP_MPMQ_CAN_AGAIN.
- * 
+ * 20211221.3 (2.5.1-dev)  Add ap_thread_create(), ap_thread_current_create()
+ *                         and ap_thread_current()
+ *
  */
 
 #define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */
@@ -708,7 +710,7 @@
 #ifndef MODULE_MAGIC_NUMBER_MAJOR
 #define MODULE_MAGIC_NUMBER_MAJOR 20211221
 #endif
-#define MODULE_MAGIC_NUMBER_MINOR 2             /* 0...n */
+#define MODULE_MAGIC_NUMBER_MINOR 3             /* 0...n */
 
 /**
  * Determine if the server's current MODULE_MAGIC_NUMBER is at least a
diff --git a/include/httpd.h b/include/httpd.h
index f9f36fee5b..60e8948b87 100644
--- a/include/httpd.h
+++ b/include/httpd.h
@@ -47,6 +47,7 @@
 #include "ap_release.h"
 
 #include "apr.h"
+#include "apr_version.h"
 #include "apr_general.h"
 #include "apr_tables.h"
 #include "apr_pools.h"
@@ -2563,6 +2564,55 @@ AP_DECLARE(void *) ap_realloc(void *ptr, size_t size)
                    AP_FN_ATTR_WARN_UNUSED_RESULT
                    AP_FN_ATTR_ALLOC_SIZE(2);
 
+#if APR_VERSION_AT_LEAST(1,8,0)
+
+/**
+ * APR 1.8+ implement those already.
+ */
+#if APR_HAS_THREAD_LOCAL
+#define AP_HAS_THREAD_LOCAL         1
+#define AP_THREAD_LOCAL             APR_THREAD_LOCAL
+#else
+#define AP_HAS_THREAD_LOCAL         0
+#endif
+#define ap_thread_create            apr_thread_create
+#define ap_thread_current           apr_thread_current
+#define ap_thread_current_create    apr_thread_current_create
+
+#else /* !APR_VERSION_AT_LEAST(1,8,0) */
+
+#if APR_HAS_THREADS
+/**
+ * AP_THREAD_LOCAL keyword mapping the compiler's.
+ */
+#if defined(__cplusplus) && __cplusplus >= 201103L
+#define AP_THREAD_LOCAL thread_local
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112
+#define AP_THREAD_LOCAL _Thread_local
+#elif defined(__GNUC__) /* works for clang too */
+#define AP_THREAD_LOCAL __thread
+#elif defined(WIN32) && defined(_MSC_VER)
+#define AP_THREAD_LOCAL __declspec(thread)
+#endif
+#endif /* APR_HAS_THREADS */
+
+#ifndef AP_THREAD_LOCAL
+#define AP_HAS_THREAD_LOCAL 0
+#define ap_thread_create apr_thread_create
+#else /* AP_THREAD_LOCAL */
+#define AP_HAS_THREAD_LOCAL 1
+AP_DECLARE(apr_status_t) ap_thread_create(apr_thread_t **thread, 
+                                          apr_threadattr_t *attr, 
+                                          apr_thread_start_t func, 
+                                          void *data, apr_pool_t *pool);
+#endif /* AP_THREAD_LOCAL */
+AP_DECLARE(apr_status_t) ap_thread_current_create(apr_thread_t **current,
+                                                  apr_threadattr_t *attr,
+                                                  apr_pool_t *pool);
+AP_DECLARE(apr_thread_t *) ap_thread_current(void);
+
+#endif /* !APR_VERSION_AT_LEAST(1,8,0) */
+
 /**
  * Get server load params
  * @param ld struct to populate: -1 in fields means error
diff --git a/modules/core/mod_watchdog.c b/modules/core/mod_watchdog.c
index e7e05287a0..99ec7cfc4d 100644
--- a/modules/core/mod_watchdog.c
+++ b/modules/core/mod_watchdog.c
@@ -280,7 +280,7 @@ static apr_status_t wd_startup(ap_watchdog_t *w, apr_pool_t *p)
     }
 
     /* Start the newly created watchdog */
-    rc = apr_thread_create(&w->thread, NULL, wd_worker, w, p);
+    rc = ap_thread_create(&w->thread, NULL, wd_worker, w, p);
     if (rc == APR_SUCCESS) {
         apr_pool_pre_cleanup_register(p, w, wd_worker_cleanup);
     }
diff --git a/modules/http2/h2_workers.c b/modules/http2/h2_workers.c
index 8b53cf9ab4..1e7aebdca8 100644
--- a/modules/http2/h2_workers.c
+++ b/modules/http2/h2_workers.c
@@ -100,8 +100,8 @@ static apr_status_t activate_slot(h2_workers *workers, h2_slot *slot)
      * to the idle queue */
     apr_atomic_inc32(&workers->worker_count);
     slot->timed_out = 0;
-    rv = apr_thread_create(&slot->thread, workers->thread_attr,
-                               slot_run, slot, workers->pool);
+    rv = ap_thread_create(&slot->thread, workers->thread_attr,
+                          slot_run, slot, workers->pool);
     if (rv != APR_SUCCESS) {
         apr_atomic_dec32(&workers->worker_count);
     }
diff --git a/modules/ssl/mod_ssl_ct.c b/modules/ssl/mod_ssl_ct.c
index f583da5f18..17b673a8e5 100644
--- a/modules/ssl/mod_ssl_ct.c
+++ b/modules/ssl/mod_ssl_ct.c
@@ -1123,8 +1123,8 @@ static int daemon_thread_start(apr_pool_t *pconf, server_rec *s_main)
 
     apr_pool_create(&pdaemon, pconf);
     apr_pool_tag(pdaemon, "sct_daemon");
-    rv = apr_thread_create(&daemon_thread, NULL, sct_daemon_thread, s_main,
-                           pconf);
+    rv = ap_thread_create(&daemon_thread, NULL, sct_daemon_thread, s_main,
+                          pconf);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_CRIT, rv, s_main,
                      APLOGNO(02709) "could not create " DAEMON_THREAD_NAME 
@@ -2522,7 +2522,7 @@ static void ssl_ct_child_init(apr_pool_t *p, server_rec *s)
         exit(APEXIT_CHILDSICK);
     }
 
-    rv = apr_thread_create(&service_thread, NULL, run_service_thread, s, p);
+    rv = ap_thread_create(&service_thread, NULL, run_service_thread, s, p);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_CRIT, rv, s,
                      APLOGNO(02745) "could not create " SERVICE_THREAD_NAME
diff --git a/server/main.c b/server/main.c
index 85df0c6a3d..2ca2aea15d 100644
--- a/server/main.c
+++ b/server/main.c
@@ -339,8 +339,8 @@ static void reset_process_pconf(process_rec *process)
     apr_pool_pre_cleanup_register(process->pconf, NULL, deregister_all_hooks);
 }
 
-#if APR_HAS_THREAD_LOCAL
-static apr_status_t main_thread_exit_cleanup(void *arg)
+#if AP_HAS_THREAD_LOCAL
+static apr_status_t main_thread_cleanup(void *arg)
 {
     apr_thread_t *thd = arg;
     apr_pool_destroy(apr_thread_pool_get(thd));
@@ -399,19 +399,19 @@ static process_rec *init_process(int *argc, const char * const * *argv)
     process->argv = *argv;
     process->short_name = apr_filepath_name_get((*argv)[0]);
 
-#if APR_HAS_THREAD_LOCAL
+#if AP_HAS_THREAD_LOCAL
     /* Create an apr_thread_t for the main thread to set up its
      * Thread Local Storage. Since it's detached and it won't
      * apr_thread_exit(), destroy its pool before exiting via
      * a process->pool cleanup
      */
     {
-        apr_thread_t *main_thd;
+        apr_thread_t *main_thd = NULL;
         apr_threadattr_t *main_thd_attr = NULL;
         if (apr_threadattr_create(&main_thd_attr, process->pool)
                 || apr_threadattr_detach_set(main_thd_attr, 1)
-                || apr_thread_current_create(&main_thd, main_thd_attr,
-                                             process->pool)) {
+                || ap_thread_current_create(&main_thd, main_thd_attr,
+                                            process->pool)) {
             char ctimebuff[APR_CTIME_LEN];
             apr_ctime(ctimebuff, apr_time_now());
             fprintf(stderr, "[%s] [crit] (%d) %s: %s failed "
@@ -420,8 +420,7 @@ static process_rec *init_process(int *argc, const char * const * *argv)
             apr_terminate();
             exit(1);
         }
-        apr_pool_cleanup_register(process->pool, main_thd,
-                                  main_thread_exit_cleanup,
+        apr_pool_cleanup_register(process->pool, main_thd, main_thread_cleanup,
                                   apr_pool_cleanup_null);
     }
 #endif
diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c
index c7509ba6a8..8ccc934ab1 100644
--- a/server/mpm/event/event.c
+++ b/server/mpm/event/event.c
@@ -2566,11 +2566,11 @@ static void create_listener_thread(thread_starter * ts)
     my_info = (proc_info *) ap_malloc(sizeof(proc_info));
     my_info->pslot = my_child_num;
     my_info->tslot = -1;      /* listener thread doesn't have a thread slot */
-    rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,
-                           my_info, pruntime);
+    rv = ap_thread_create(&ts->listener, thread_attr, listener_thread,
+                          my_info, pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00474)
-                     "apr_thread_create: unable to create listener thread");
+                     "ap_thread_create: unable to create listener thread");
         /* let the parent decide how bad this really is */
         clean_child_exit(APEXIT_CHILDSICK);
     }
@@ -2761,12 +2761,12 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
             /* We let each thread update its own scoreboard entry.  This is
              * done because it lets us deal with tid better.
              */
-            rv = apr_thread_create(&threads[i], thread_attr,
-                                   worker_thread, my_info, pruntime);
+            rv = ap_thread_create(&threads[i], thread_attr,
+                                  worker_thread, my_info, pruntime);
             if (rv != APR_SUCCESS) {
                 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf,
                              APLOGNO(03104)
-                             "apr_thread_create: unable to create worker thread");
+                             "ap_thread_create: unable to create worker thread");
                 /* let the parent decide how bad this really is */
                 clean_child_exit(APEXIT_CHILDSICK);
             }
@@ -2880,6 +2880,15 @@ static void join_start_thread(apr_thread_t * start_thread_id)
     }
 }
 
+#if AP_HAS_THREAD_LOCAL
+static apr_status_t main_thread_cleanup(void *arg)
+{
+    apr_thread_t *thd = arg;
+    apr_pool_destroy(apr_thread_pool_get(thd));
+    return APR_SUCCESS;
+}
+#endif
+
 static void child_main(int child_num_arg, int child_bucket)
 {
     apr_thread_t **threads;
@@ -2902,6 +2911,28 @@ static void child_main(int child_num_arg, int child_bucket)
     apr_pool_create(&pchild, pconf);
     apr_pool_tag(pchild, "pchild");
 
+#if AP_HAS_THREAD_LOCAL
+    /* Create an apr_thread_t for the main child thread to set up its
+     * Thread Local Storage. Since it's detached and it won't
+     * apr_thread_exit(), destroy its pool before exiting via
+     * a pchild cleanup
+     */
+    if (!one_process) {
+        apr_thread_t *main_thd = NULL;
+        apr_threadattr_t *main_thd_attr = NULL;
+        if (apr_threadattr_create(&main_thd_attr, pchild)
+                || apr_threadattr_detach_set(main_thd_attr, 1)
+                || ap_thread_current_create(&main_thd, main_thd_attr,
+                                            pchild)) {
+            ap_log_error(APLOG_MARK, APLOG_EMERG, 0, ap_server_conf, APLOGNO()
+                         "Couldn't initialize child main thread");
+            clean_child_exit(APEXIT_CHILDFATAL);
+        }
+        apr_pool_cleanup_register(pchild, main_thd, main_thread_cleanup,
+                                  apr_pool_cleanup_null);
+    }
+#endif
+
     /* close unused listeners and pods */
     for (i = 0; i < retained->mpm->num_buckets; i++) {
         if (i != child_bucket) {
@@ -2974,11 +3005,11 @@ static void child_main(int child_num_arg, int child_bucket)
     ts->child_num_arg = child_num_arg;
     ts->threadattr = thread_attr;
 
-    rv = apr_thread_create(&start_thread_id, thread_attr, start_threads,
-                           ts, pchild);
+    rv = ap_thread_create(&start_thread_id, thread_attr, start_threads,
+                          ts, pchild);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00480)
-                     "apr_thread_create: unable to create worker thread");
+                     "ap_thread_create: unable to create worker thread");
         /* let the parent decide how bad this really is */
         clean_child_exit(APEXIT_CHILDSICK);
     }
diff --git a/server/mpm/prefork/prefork.c b/server/mpm/prefork/prefork.c
index ef5685fab9..de7c5b1fe9 100644
--- a/server/mpm/prefork/prefork.c
+++ b/server/mpm/prefork/prefork.c
@@ -398,11 +398,19 @@ static void child_sigmask(sigset_t *new_mask, sigset_t *old_mask)
 }
 #endif
 
+#if AP_HAS_THREAD_LOCAL
+static apr_status_t main_thread_cleanup(void *arg)
+{
+    apr_thread_t *thd = arg;
+    apr_pool_destroy(apr_thread_pool_get(thd));
+    return APR_SUCCESS;
+}
+#endif
+
 static void child_main(int child_num_arg, int child_bucket)
 {
 #if APR_HAS_THREADS
     apr_thread_t *thd = NULL;
-    apr_os_thread_t osthd;
     sigset_t sig_mask;
 #endif
     apr_pool_t *ptrans;
@@ -434,9 +442,36 @@ static void child_main(int child_num_arg, int child_bucket)
     apr_allocator_owner_set(allocator, pchild);
     apr_pool_tag(pchild, "pchild");
 
+#if AP_HAS_THREAD_LOCAL
+    /* Create an apr_thread_t for the main child thread to set up its
+     * Thread Local Storage. Since it's detached and it won't
+     * apr_thread_exit(), destroy its pool before exiting via
+     * a pchild cleanup
+     */
+    if (one_process) {
+        thd = ap_thread_current();
+    }
+    else {
+        apr_threadattr_t *main_thd_attr = NULL;
+        if (apr_threadattr_create(&main_thd_attr, pchild)
+                || apr_threadattr_detach_set(main_thd_attr, 1)
+                || ap_thread_current_create(&thd, main_thd_attr,
+                                            pchild)) {
+            ap_log_error(APLOG_MARK, APLOG_EMERG, 0, ap_server_conf, APLOGNO()
+                         "Couldn't initialize child main thread");
+            clean_child_exit(APEXIT_CHILDFATAL);
+        }
+        apr_pool_cleanup_register(pchild, thd, main_thread_cleanup,
+                                  apr_pool_cleanup_null);
+    }
+#elif APR_HAS_THREADS
+    {
+        apr_os_thread_t osthd = apr_os_thread_current();
+        apr_os_thread_put(&thd, &osthd, pchild);
+    }
+#endif
 #if APR_HAS_THREADS
-    osthd = apr_os_thread_current();
-    apr_os_thread_put(&thd, &osthd, pchild);
+    ap_assert(thd != NULL);
 #endif
 
     apr_pool_create(&ptrans, pchild);
diff --git a/server/mpm/winnt/child.c b/server/mpm/winnt/child.c
index fd63cfc666..52ff30ff51 100644
--- a/server/mpm/winnt/child.c
+++ b/server/mpm/winnt/child.c
@@ -778,24 +778,27 @@ static winnt_conn_ctx_t *winnt_get_connection(winnt_conn_ctx_t *context)
     return context;
 }
 
+struct worker_info {
+    apr_thread_t *thd;
+    HANDLE handle;
+    int num;
+};
+
 /*
- * worker_main()
+ * worker_thread()
  * Main entry point for the worker threads. Worker threads block in
  * win*_get_connection() awaiting a connection to service.
  */
-static DWORD __stdcall worker_main(void *thread_num_val)
+static void *APR_THREAD_FUNC worker_thread(apr_thread_t *thd, void *ctx)
 {
-    apr_thread_t *thd;
-    apr_os_thread_t osthd;
+    struct worker_info *info = ctx;
     static int requests_this_child = 0;
     winnt_conn_ctx_t *context = NULL;
-    int thread_num = (int)thread_num_val;
+    int thread_num = info->num;
     ap_sb_handle_t *sbh;
     conn_rec *c;
     apr_int32_t disconnected;
 
-    osthd = apr_os_thread_current();
-
     while (1) {
 
         ap_update_child_status_from_indexes(0, thread_num, SERVER_READY, NULL);
@@ -827,8 +830,6 @@ static DWORD __stdcall worker_main(void *thread_num_val)
             continue;
         }
 
-        thd = NULL;
-        apr_os_thread_put(&thd, &osthd, context->ptrans);
         c->current_thread = thd;
 
         ap_process_connection(c, context->sock);
@@ -843,6 +844,7 @@ static DWORD __stdcall worker_main(void *thread_num_val)
 
     ap_update_child_status_from_indexes(0, thread_num, SERVER_DEAD, NULL);
 
+    SetEvent(info->handle);
     return 0;
 }
 
@@ -889,13 +891,21 @@ static void create_listener_thread(void)
 }
 
 
+#if AP_HAS_THREAD_LOCAL
+static apr_status_t main_thread_cleanup(void *arg)
+{
+    apr_thread_t *thd = arg;
+    apr_pool_destroy(apr_thread_pool_get(thd));
+    return APR_SUCCESS;
+}
+#endif
+
 void child_main(apr_pool_t *pconf, DWORD parent_pid)
 {
     apr_status_t status;
-    apr_hash_t *ht;
     ap_listen_rec *lr;
     HANDLE child_events[3];
-    HANDLE *child_handles;
+    struct worker_info *workers;
     int listener_started = 0;
     int threads_created = 0;
     int time_remains;
@@ -911,8 +921,29 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
     apr_pool_create(&pchild, pconf);
     apr_pool_tag(pchild, "pchild");
 
+#if AP_HAS_THREAD_LOCAL
+    /* Create an apr_thread_t for the main child thread to set up its
+     * Thread Local Storage. Since it's detached and it won't
+     * apr_thread_exit(), destroy its pool before exiting via
+     * a pchild cleanup
+     */
+    {
+        apr_thread_t *main_thd = NULL;
+        apr_threadattr_t *main_thd_attr = NULL;
+        if (apr_threadattr_create(&main_thd_attr, pchild)
+                || apr_threadattr_detach_set(main_thd_attr, 1)
+                || ap_thread_current_create(&main_thd, main_thd_attr,
+                                            pchild)) {
+            ap_log_error(APLOG_MARK, APLOG_EMERG, 0, ap_server_conf, APLOGNO()
+                         "Couldn't initialize child main thread");
+            exit(APEXIT_CHILDINIT);
+        }
+        apr_pool_cleanup_register(pchild, main_thd, main_thread_cleanup,
+                                  apr_pool_cleanup_null);
+    }
+#endif
+
     ap_run_child_init(pchild, ap_server_conf);
-    ht = apr_hash_make(pchild);
 
     listener_shutdown_event = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!listener_shutdown_event) {
@@ -983,15 +1014,13 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
      */
     ap_log_error(APLOG_MARK, APLOG_NOTICE, APR_SUCCESS, ap_server_conf, APLOGNO(00354)
                  "Child: Starting %d worker threads.", ap_threads_per_child);
-    child_handles = (HANDLE) apr_pcalloc(pchild, ap_threads_per_child
-                                                  * sizeof(HANDLE));
+    workers = apr_pcalloc(pchild, ap_threads_per_child * sizeof(*workers));
     apr_thread_mutex_create(&child_lock, APR_THREAD_MUTEX_DEFAULT, pchild);
 
     while (1) {
         int from_previous_generation = 0, starting_up = 0;
 
         for (i = 0; i < ap_threads_per_child; i++) {
-            int *score_idx;
             int status = ap_scoreboard_image->servers[0][i].status;
             if (status != SERVER_GRACEFUL && status != SERVER_DEAD) {
                 if (ap_scoreboard_image->servers[0][i].generation != my_generation) {
@@ -1004,13 +1033,24 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
             }
             ap_update_child_status_from_indexes(0, i, SERVER_STARTING, NULL);
 
-            child_handles[i] = CreateThread(NULL, ap_thread_stacksize,
-                                            worker_main, (void *) i,
-                                            stack_res_flag, &tid);
-            if (child_handles[i] == 0) {
-                ap_log_error(APLOG_MARK, APLOG_CRIT, apr_get_os_error(),
-                             ap_server_conf, APLOGNO(00355)
-                             "Child: CreateThread failed. Unable to "
+            workers[i].num = i;
+            workers[i].handle = CreateEvent(NULL, TRUE, FALSE, NULL);
+            if (!workers[i].handle) {
+                rv = apr_get_os_error();
+            }
+            else {
+                apr_threadattr_t *thread_attr = NULL;
+                apr_threadattr_create(&thread_attr, pchild);
+                if (ap_thread_stacksize != 0) {
+                    apr_threadattr_stacksize_set(thread_attr,
+                                                 ap_thread_stacksize);
+                }
+                rv = ap_thread_create(&workers[i].thd, thread_attr,
+                                      worker_thread, &workers[i], pchild);
+            }
+            if (rv != APR_SUCCESS) {
+                ap_log_error(APLOG_MARK, APLOG_CRIT, rv, ap_server_conf, APLOGNO(00355)
+                             "Child: thread creation failed. Unable to "
                              "create all worker threads. Created %d of the %d "
                              "threads requested with the ThreadsPerChild "
                              "configuration directive.",
@@ -1021,14 +1061,6 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
             ap_scoreboard_image->servers[0][i].pid = my_pid;
             ap_scoreboard_image->servers[0][i].generation = my_generation;
             threads_created++;
-            /* Save the score board index in ht keyed to the thread handle.
-             * We need this when cleaning up threads down below...
-             */
-            apr_thread_mutex_lock(child_lock);
-            score_idx = apr_pcalloc(pchild, sizeof(int));
-            *score_idx = i;
-            apr_hash_set(ht, &child_handles[i], sizeof(HANDLE), score_idx);
-            apr_thread_mutex_unlock(child_lock);
         }
         /* Start the listener only when workers are available */
         if (!listener_started && threads_created) {
@@ -1199,7 +1231,7 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
 
     while (threads_created)
     {
-        HANDLE handle = child_handles[threads_created - 1];
+        struct worker_info *info = workers[threads_created - 1];
         DWORD dwRet;
 
         if (time_remains < 0)
@@ -1213,13 +1245,15 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
                          time_remains / 1000, threads_created);
         }
 
-        dwRet = WaitForSingleObject(handle, 100);
+        dwRet = WaitForSingleObject(info->handle, 100);
         time_remains -= 100;
         if (dwRet == WAIT_TIMEOUT) {
             /* Keep waiting */
         }
         else if (dwRet == WAIT_OBJECT_0) {
-            CloseHandle(handle);
+            apr_status_t thread_rv;
+            apr_thread_join(&thread_rv, info->thd);
+            CloseHandle(info->handle);
             threads_created--;
         }
         else {
@@ -1232,11 +1266,8 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
                      "Child: Waiting for %d threads timed out, terminating process.",
                      threads_created);
         for (i = 0; i < threads_created; i++) {
-            /* Reset the scoreboard entries for the threads. */
-            int *idx = apr_hash_get(ht, &child_handles[i], sizeof(HANDLE));
-            if (idx) {
-                ap_update_child_status_from_indexes(0, *idx, SERVER_DEAD, NULL);
-            }
+            struct worker_info *info = workers[i];
+            ap_update_child_status_from_indexes(0, info->num, SERVER_DEAD, NULL);
         }
         /* We can't wait for any longer, but still have some threads remaining.
          *
diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c
index ddcde0f879..3266130d5a 100644
--- a/server/mpm/worker/worker.c
+++ b/server/mpm/worker/worker.c
@@ -855,11 +855,11 @@ static void create_listener_thread(thread_starter *ts)
     my_info->pid = my_child_num;
     my_info->tid = -1; /* listener thread doesn't have a thread slot */
     my_info->sd = 0;
-    rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,
-                           my_info, pruntime);
+    rv = ap_thread_create(&ts->listener, thread_attr, listener_thread,
+                          my_info, pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00275)
-                     "apr_thread_create: unable to create listener thread");
+                     "ap_thread_create: unable to create listener thread");
         /* let the parent decide how bad this really is */
         clean_child_exit(APEXIT_CHILDSICK);
     }
@@ -975,11 +975,11 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
             /* We let each thread update its own scoreboard entry.  This is
              * done because it lets us deal with tid better.
              */
-            rv = apr_thread_create(&threads[i], thread_attr,
-                                   worker_thread, my_info, pruntime);
+            rv = ap_thread_create(&threads[i], thread_attr,
+                                  worker_thread, my_info, pruntime);
             if (rv != APR_SUCCESS) {
                 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03142)
-                             "apr_thread_create: unable to create worker thread");
+                             "ap_thread_create: unable to create worker thread");
                 /* let the parent decide how bad this really is */
                 clean_child_exit(APEXIT_CHILDSICK);
             }
@@ -1102,6 +1102,15 @@ static void join_start_thread(apr_thread_t *start_thread_id)
     }
 }
 
+#if AP_HAS_THREAD_LOCAL
+static apr_status_t main_thread_cleanup(void *arg)
+{
+    apr_thread_t *thd = arg;
+    apr_pool_destroy(apr_thread_pool_get(thd));
+    return APR_SUCCESS;
+}
+#endif
+
 static void child_main(int child_num_arg, int child_bucket)
 {
     apr_thread_t **threads;
@@ -1123,6 +1132,28 @@ static void child_main(int child_num_arg, int child_bucket)
     apr_pool_create(&pchild, pconf);
     apr_pool_tag(pchild, "pchild");
 
+#if AP_HAS_THREAD_LOCAL
+    /* Create an apr_thread_t for the main child thread to set up its
+     * Thread Local Storage. Since it's detached and it won't
+     * apr_thread_exit(), destroy its pool before exiting via
+     * a pchild cleanup
+     */
+    if (!one_process) {
+        apr_thread_t *main_thd = NULL;
+        apr_threadattr_t *main_thd_attr = NULL;
+        if (apr_threadattr_create(&main_thd_attr, pchild)
+                || apr_threadattr_detach_set(main_thd_attr, 1)
+                || ap_thread_current_create(&main_thd, main_thd_attr,
+                                            pchild)) {
+            ap_log_error(APLOG_MARK, APLOG_EMERG, 0, ap_server_conf, APLOGNO()
+                         "Couldn't initialize child main thread");
+            clean_child_exit(APEXIT_CHILDFATAL);
+        }
+        apr_pool_cleanup_register(pchild, main_thd, main_thread_cleanup,
+                                  apr_pool_cleanup_null);
+    }
+#endif
+
     /* close unused listeners and pods */
     for (i = 0; i < retained->mpm->num_buckets; i++) {
         if (i != child_bucket) {
@@ -1202,11 +1233,11 @@ static void child_main(int child_num_arg, int child_bucket)
     ts->child_num_arg = child_num_arg;
     ts->threadattr = thread_attr;
 
-    rv = apr_thread_create(&start_thread_id, thread_attr, start_threads,
-                           ts, pchild);
+    rv = ap_thread_create(&start_thread_id, thread_attr, start_threads,
+                          ts, pchild);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00282)
-                     "apr_thread_create: unable to create worker thread");
+                     "ap_thread_create: unable to create worker thread");
         /* let the parent decide how bad this really is */
         clean_child_exit(APEXIT_CHILDSICK);
     }
diff --git a/server/util.c b/server/util.c
index 9d056627af..39bde9c98e 100644
--- a/server/util.c
+++ b/server/util.c
@@ -3261,6 +3261,86 @@ AP_DECLARE(void *) ap_realloc(void *ptr, size_t size)
     return p;
 }
 
+#if !APR_VERSION_AT_LEAST(1,8,0)
+
+#if AP_HAS_THREAD_LOCAL
+struct thread_ctx {
+    apr_thread_start_t func;
+    void *data;
+};
+
+static AP_THREAD_LOCAL apr_thread_t *current_thread = NULL;
+
+static void *APR_THREAD_FUNC thread_start(apr_thread_t *thread, void *data)
+{
+    struct thread_ctx *ctx = data;
+
+    current_thread = thread;
+    return ctx->func(thread, ctx->data);
+}
+
+AP_DECLARE(apr_status_t) ap_thread_create(apr_thread_t **thread, 
+                                          apr_threadattr_t *attr, 
+                                          apr_thread_start_t func, 
+                                          void *data, apr_pool_t *pool)
+{
+    struct thread_ctx *ctx = apr_palloc(pool, sizeof(*ctx));
+
+    ctx->func = func;
+    ctx->data = data;
+    return apr_thread_create(thread, attr, thread_start, ctx, pool);
+}
+#endif /* AP_HAS_THREAD_LOCAL */
+
+AP_DECLARE(apr_status_t) ap_thread_current_create(apr_thread_t **current,
+                                                  apr_threadattr_t *attr,
+                                                  apr_pool_t *pool)
+{
+    apr_status_t rv;
+    apr_os_thread_t osthd;
+    apr_abortfunc_t abort_fn = apr_pool_abort_get(pool);
+    apr_allocator_t *allocator;
+    apr_pool_t *p;
+
+    *current = NULL;
+
+    rv = apr_allocator_create(&allocator);
+    if (rv != APR_SUCCESS) {
+        if (abort_fn)
+            abort_fn(rv);
+        return rv;
+    }
+    rv = apr_pool_create_unmanaged_ex(&p, abort_fn, allocator);
+    if (rv != APR_SUCCESS) {
+        apr_allocator_destroy(allocator);
+        return rv;
+    }
+    apr_allocator_owner_set(allocator, p);
+
+    osthd = apr_os_thread_current();
+    rv = apr_os_thread_put(current, &osthd, p);
+    if (rv != APR_SUCCESS) {
+        apr_pool_destroy(p);
+        return rv;
+    }
+
+#if AP_HAS_THREAD_LOCAL
+    current_thread = *current;
+#endif
+    return APR_SUCCESS;
+}
+
+AP_DECLARE(apr_thread_t *) ap_thread_current(void)
+{
+#if AP_HAS_THREAD_LOCAL
+    return current_thread;
+#else
+    return NULL;
+#endif
+}
+
+#endif /* !APR_VERSION_AT_LEAST(1,8,0) */
+
 AP_DECLARE(void) ap_get_sload(ap_sload_t *ld)
 {
     int i, j, server_limit, thread_limit;
diff --git a/server/util_pcre.c b/server/util_pcre.c
index 0e48448b67..68224e7cbf 100644
--- a/server/util_pcre.c
+++ b/server/util_pcre.c
@@ -269,7 +269,7 @@ AP_DECLARE(int) ap_regcomp(ap_regex_t * preg, const char *pattern, int cflags)
  * context per thread (in Thread Local Storage, TLS) grown as needed, and while
  * at it we do the same for PCRE1 ints vectors. Note that this requires a fast
  * TLS mechanism to be worth it, which is the case of apr_thread_data_get/set()
- * from/to apr_thread_current() when APR_HAS_THREAD_LOCAL; otherwise we'll do
+ * from/to ap_thread_current() when AP_HAS_THREAD_LOCAL; otherwise we'll do
  * the allocation and freeing for each ap_regexec().
  */
 
@@ -318,7 +318,7 @@ void free_match_data(match_data_pt data, apr_size_t size)
 #endif
 }
 
-#if APR_HAS_THREAD_LOCAL
+#if AP_HAS_THREAD_LOCAL
 
 struct apreg_tls {
     match_data_pt data;
@@ -342,10 +342,10 @@ static match_data_pt get_match_data(apr_size_t size,
     apr_thread_t *current;
     struct apreg_tls *tls = NULL;
 
-    /* Even though APR_HAS_THREAD_LOCAL, we may still be called by a
+    /* Even though AP_HAS_THREAD_LOCAL, we may still be called by a
      * native/non-apr thread, let's fall back to alloc/free in this case.
      */
-    current = apr_thread_current();
+    current = ap_thread_current();
     if (!current) {
         *to_free = 1;
         return alloc_match_data(size, ovector, small_vector);
@@ -391,7 +391,7 @@ static match_data_pt get_match_data(apr_size_t size,
     return tls->data;
 }
 
-#else /* !APR_HAS_THREAD_LOCAL */
+#else /* !AP_HAS_THREAD_LOCAL */
 
 static APR_INLINE match_data_pt get_match_data(apr_size_t size,
                                                match_vector_pt *ovector,
@@ -402,7 +402,7 @@ static APR_INLINE match_data_pt get_match_data(apr_size_t size,
     return alloc_match_data(size, ovector, small_vector);
 }
 
-#endif /* !APR_HAS_THREAD_LOCAL */
+#endif /* !AP_HAS_THREAD_LOCAL */
 
 AP_DECLARE(int) ap_regexec(const ap_regex_t *preg, const char *string,
                            apr_size_t nmatch, ap_regmatch_t *pmatch,