Merge r582543, r1090093, r1102978, r1909474, r1909585, r1909586, r1909590, r1909677 from trunk:1.7.x

In preparation to be able to test multiple, parallel
flavors of the apr build, we'll need to designate the
path of the apr-invoked binaires.  Macroize this.

Removed tabs and trailing spaces.


Appears to be required for getpid()


In case that we have threads store the apr_reslist parameters in the
apr_memcache_server_t struct for later usage.

* include/apr_memcache.h::struct apr_memcache_server_t:
    Add missing fields to struct.

* memcache/apr_memcache.c::apr_memcache_server_create:
    Init fields with given values.


Check sockets from connection pool before using them and try to reconnect
them if they are not usable any longer.

* memcache/apr_memcache.c::ms_find_conn:
    Check if the socket returned from the connection pool is still readable. If
    not then invalidate the connection in the pool and request a new one
    from the connection pool. Repeat this until a valid socket is returned
    or this was done the maximum number of connections in the pool plus one.
    This ensures that at least one new socket was created. If a new socket
    does not work this indicates a broken backend and not just a restart in
    the past. In this case return an error like previously.

* test/testmemcache.c:
    Add new test for connection validation.

* test/memcachedmock.c:
    For the new test we need a memcached mock server that we control and
    can restart.

* test/testmemcache.h:
    Shared defines between test/testmemcache.c and test/memcachedmock.c.

* test/Makefile.in:
* test/Makefile.win:
* test/NWGNUmakefile:
* test/NWGNUmemcachedmock:
    Needed changes to build test/memcachedmock.c on different platforms.


* Add missing test/memcachedmock.c

* Wait for the mock memcached to shutdown the socket

* Add CHANGES entry for r1909474, r1909585, r1909586, r1909590
Reviewed by: ruediger


git-svn-id: https://svn.apache.org/repos/asf/apr/apr-util/branches/1.7.x@1909699 13f79535-47bb-0310-9956-ffa450edef68

12 files changed, 611 insertions, 73 deletions

diff --git a/CHANGES b/CHANGES
index af673906..a0048d1f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,9 @@
                                                      -*- coding: utf-8 -*-
 Changes with APR-util 1.7.0
 
+  *) apr_memcache: Check sockets from connection pool before using them and try
+     to reconnect them if they are not usable any longer.[Ruediger Pluem]
+
   *) apr_crypto_openssl: Compatibility with OpenSSL 3.  [Yann Ylavic]
 
   *) configure: Fix configure for compilers which don't accept implicit
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 26b74b56..6c3b6a5a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -351,6 +351,12 @@ IF(APR_BUILD_TESTAPR)
     ADD_TEST(NAME dbd-${somedbd} COMMAND dbd ${somedbd})
   ENDFOREACH()
 
+  ADD_EXECUTABLE(memcachedmock test/memcachedmock)
+  TARGET_LINK_LIBRARIES(memcachedmock ${whichapr})
+  IF(apiflag)
+    SET_TARGET_PROPERTIES(dbd PROPERTIES COMPILE_FLAGS ${apiflag})
+  ENDIF()
+
 ENDIF (APR_BUILD_TESTAPR)
 
 # Installation
diff --git a/include/apr_memcache.h b/include/apr_memcache.h
index 82878825..6ffd6622 100644
--- a/include/apr_memcache.h
+++ b/include/apr_memcache.h
@@ -72,6 +72,13 @@ struct apr_memcache_server_t
     apr_thread_mutex_t *lock;
 #endif
     apr_time_t btime;
+#if APR_HAS_THREADS
+    /** Resource list parameters */
+    apr_uint32_t min;
+    apr_uint32_t smax;
+    apr_uint32_t max;
+    apr_uint32_t ttl;
+#endif
 };
 
 /* Custom hash callback function prototype, user for server selection.
diff --git a/memcache/apr_memcache.c b/memcache/apr_memcache.c
index ab5ea83d..1d3ad2cd 100644
--- a/memcache/apr_memcache.c
+++ b/memcache/apr_memcache.c
@@ -222,22 +222,83 @@ APU_DECLARE(apr_memcache_server_t *) apr_memcache_find_server(apr_memcache_t *mc
     return NULL;
 }
 
-static apr_status_t ms_find_conn(apr_memcache_server_t *ms, apr_memcache_conn_t **conn) 
+
+/* Forward declare mc_conn_construct */
+static apr_status_t
+mc_conn_construct(void **conn_, void *params, apr_pool_t *pool);
+
+static apr_status_t ms_find_conn(apr_memcache_server_t *ms, apr_memcache_conn_t **conn)
 {
-    apr_status_t rv;
+    apr_status_t rv = APR_SUCCESS;
     apr_bucket_alloc_t *balloc;
     apr_bucket *e;
+#if APR_HAS_THREADS
+    int i;
+#endif
+    int atreadeof;
 
 #if APR_HAS_THREADS
-    rv = apr_reslist_acquire(ms->conns, (void **)conn);
+    /*
+     * In order to avoid an endless loop in case that a freshly connected
+     * socket is immediately closed by the remote side we limit this loop
+     * to the maxium number of connections in the reslist plus 1.
+     */
+    for (i = 0; i <= ms->max; i++) {
+        rv = apr_reslist_acquire(ms->conns, (void **)conn);
+
+        if (rv != APR_SUCCESS) {
+            return rv;
+        }
+
+        atreadeof = 0;
+        rv = apr_socket_atreadeof((*conn)->sock, &atreadeof);
+
+        if ((rv == APR_SUCCESS) && !atreadeof) {
+            break;
+        }
+        /*
+         * The socket we got is fishy. But maybe the memcached was just
+         * restarted. Hence give it a chance by destroying the socket and
+         * getting a new one.
+         */
+        rv = apr_reslist_invalidate(ms->conns, *conn);
+        *conn = NULL;
+
+        if (rv != APR_SUCCESS) {
+            return rv;
+        }
+    }
+
+    /*
+     * Even after refreshing all sockets we still do not have a working one.
+     * Give up.
+     */
+    if (!(*conn)) {
+        return APR_EGENERAL;
+    }
 #else
     *conn = ms->conn;
-    rv = APR_SUCCESS;
-#endif
-
-    if (rv != APR_SUCCESS) {
-        return rv;
+    atreadeof = 0;
+    if (*conn) {
+        rv = apr_socket_atreadeof((*conn)->sock, &atreadeof);
+        if ((rv != APR_SUCCESS) || !atreadeof) {
+            ms->conn = NULL;
+            apr_pool_destroy((*conn)->p);
+            rv = mc_conn_construct((void**)conn, ms, ms->p);
+            if (rv != APR_SUCCESS) {
+                return rv;
+            }
+            ms->conn = *conn;
+        }
+    }
+    else {
+        rv = mc_conn_construct((void**)conn, ms, ms->p);
+        if (rv != APR_SUCCESS) {
+            return rv;
+        }
+        ms->conn = *conn;
     }
+#endif
 
     balloc = apr_bucket_alloc_create((*conn)->tp);
     (*conn)->bb = apr_brigade_create((*conn)->tp, balloc);
@@ -432,6 +493,11 @@ APU_DECLARE(apr_status_t) apr_memcache_server_create(apr_pool_t *p,
         return rv;
     }
 
+    server->min = min;
+    server->smax = smax;
+    server->max = max;
+    server->ttl = ttl;
+
     apr_reslist_cleanup_order_set(server->conns, APR_RESLIST_CLEANUP_FIRST);
 #else
     rv = mc_conn_construct((void**)&(server->conn), server, np);
diff --git a/test/Makefile.in b/test/Makefile.in
index 88270763..cd0c4bf4 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -57,6 +57,10 @@ OBJECTS_dbd = dbd.lo $(LOCAL_LIBS)
 dbd: $(OBJECTS_dbd)
 	$(LINK_PROG) $(OBJECTS_dbd) $(APRUTIL_LIBS)
 
+OBJECTS_memcachedmock = memcachedmock.lo $(LOCAL_LIBS)
+memcachedmock: $(OBJECTS_memcachedmock)
+	$(LINK_PROG) $(OBJECTS_memcachedmock) $(APRUTIL_LIBS)
+
 check: $(TESTALL_COMPONENTS) $(STDTEST_PORTABLE) $(STDTEST_NONPORTABLE)
 	teststatus=0; \
 	progfailed=""; \
diff --git a/test/Makefile.win b/test/Makefile.win
index fa3d2eba..b0293bcf 100644
--- a/test/Makefile.win
+++ b/test/Makefile.win
@@ -141,6 +141,10 @@ $(OUTDIR)\dbd.exe: $(INTDIR)\dbd.obj $(PROGRAM_DEPENDENCIES)
 	@if exist "$@.manifest" \
 	    mt.exe -manifest "$@.manifest" -outputresource:$@;1
 
+$(OUTDIR)\memcachedmock.exe: $(INTDIR)\memcachedmock.obj $(PROGRAM_DEPENDENCIES)
+	$(LD) $(LDFLAGS) /out:"$@" $** $(LD_LIBS)
+	@if exist "$@.manifest" \
+	    mt.exe -manifest "$@.manifest" -outputresource:$@;1
 
 cleandata:
 	@for %f in ($(CLEAN_DATA)) do @if EXIST %f del /f %f
diff --git a/test/NWGNUmakefile b/test/NWGNUmakefile
index a955f2c0..44401935 100644
--- a/test/NWGNUmakefile
+++ b/test/NWGNUmakefile
@@ -166,6 +166,7 @@ XDCDATA		=
 TARGET_nlm = \
 	$(OBJDIR)/aputest.nlm \
 	$(OBJDIR)/aputest.nlm \
+	$(OBJDIR)/memcachedmock.nlm \
 	$(EOLIST)
 #
 # If there is an LIB target, put it here
diff --git a/test/NWGNUmemcachedmock b/test/NWGNUmemcachedmock
new file mode 100644
index 00000000..db35f34b
--- /dev/null
+++ b/test/NWGNUmemcachedmock
@@ -0,0 +1,252 @@
+#
+# Make sure all needed macro's are defined
+#
+
+#
+# Get the 'head' of the build environment if necessary.  This includes default
+# targets and paths to tools
+#
+
+ifndef EnvironmentDefined
+include $(APR_WORK)/build/NWGNUhead.inc
+endif
+
+#
+# These directories will be at the beginning of the include list, followed by
+# INCDIRS
+#
+XINCDIRS	+= \
+			$(APR)/include \
+			$(APR)/include/arch/netware \
+			$(EOLIST)
+
+#
+# These flags will come after CFLAGS
+#
+XCFLAGS		+= \
+			$(EOLIST)
+
+#
+# These defines will come after DEFINES
+#
+XDEFINES	+= \
+			$(EOLIST)
+
+#
+# These flags will be added to the link.opt file
+#
+XLFLAGS		+= \
+			$(EOLIST)
+
+#
+# These values will be appended to the correct variables based on the value of
+# RELEASE
+#
+ifeq "$(RELEASE)" "debug"
+XINCDIRS	+= \
+			$(EOLIST)
+
+XCFLAGS		+= \
+			$(EOLIST)
+
+XDEFINES	+= \
+			$(EOLIST)
+
+XLFLAGS		+= \
+			$(EOLIST)
+endif
+
+ifeq "$(RELEASE)" "noopt"
+XINCDIRS	+= \
+			$(EOLIST)
+
+XCFLAGS		+= \
+			$(EOLIST)
+
+XDEFINES	+= \
+			$(EOLIST)
+
+XLFLAGS		+= \
+			$(EOLIST)
+endif
+
+ifeq "$(RELEASE)" "release"
+XINCDIRS	+= \
+			$(EOLIST)
+
+XCFLAGS		+= \
+			$(EOLIST)
+
+XDEFINES	+= \
+			$(EOLIST)
+
+XLFLAGS		+= \
+			$(EOLIST)
+endif
+
+#
+# These are used by the link target if an NLM is being generated
+# This is used by the link 'name' directive to name the nlm.  If left blank
+# TARGET_nlm (see below) will be used.
+#
+NLM_NAME	= memcachedmock
+
+#
+# This is used by the link '-desc ' directive. 
+# If left blank, NLM_NAME will be used.
+#
+NLM_DESCRIPTION	= socket NLM to test sockets
+
+#
+# This is used by the '-threadname' directive.  If left blank,
+# NLM_NAME Thread will be used.
+#
+NLM_THREAD_NAME	= $(NLM_NAME)
+
+#
+# This is used by the '-screenname' directive.  If left blank,
+# 'Apache for NetWare' Thread will be used.
+#
+NLM_SCREEN_NAME = DEFAULT
+
+#
+# If this is specified, it will override VERSION value in 
+# $(APR_WORK)/build/NWGNUenvironment.inc
+#
+NLM_VERSION	=
+
+#
+# If this is specified, it will override the default of 64K
+#
+NLM_STACK_SIZE	= 
+
+#
+# If this is specified it will be used by the link '-entry' directive
+#
+NLM_ENTRY_SYM	=
+
+#
+# If this is specified it will be used by the link '-exit' directive
+#
+NLM_EXIT_SYM	=
+
+#
+# If this is specified it will be used by the link '-check' directive
+#
+NLM_CHECK_SYM	=
+
+#
+# If this is specified it will be used by the link '-flags' directive
+#
+NLM_FLAGS	= AUTOUNLOAD, PSEUDOPREEMPTION, MULTIPLE
+ 
+#
+# If this is specified it will be linked in with the XDCData option in the def 
+# file instead of the default of $(APR)/misc/netware/apache.xdc.  XDCData can 
+# be disabled by setting APACHE_UNIPROC in the environment
+#
+XDCDATA		= 
+
+#
+# Declare all target files (you must add your files here)
+#
+
+#
+# If there is an NLM target, put it here
+#
+TARGET_nlm = \
+	$(OBJDIR)/$(NLM_NAME).nlm \
+	$(EOLIST)
+
+#
+# If there is an LIB target, put it here
+#
+TARGET_lib = \
+	$(EOLIST)
+
+#
+# These are the OBJ files needed to create the NLM target above.
+# Paths must all use the '/' character
+#
+FILES_nlm_objs = \
+	$(OBJDIR)/$(NLM_NAME).o \
+	$(EOLIST)
+
+#
+# These are the LIB files needed to create the NLM target above.
+# These will be added as a library command in the link.opt file.
+#
+FILES_nlm_libs = \
+	$(PRELUDE) \
+	$(EOLIST)
+
+#
+# These are the modules that the above NLM target depends on to load.
+# These will be added as a module command in the link.opt file.
+#
+FILES_nlm_modules = \
+	aprlib \
+	libc \
+	$(EOLIST)
+
+#
+# If the nlm has a msg file, put it's path here
+#
+FILE_nlm_msg =
+ 
+#
+# If the nlm has a hlp file put it's path here
+#
+FILE_nlm_hlp =
+
+#
+# If this is specified, it will override the default copyright.
+#
+FILE_nlm_copyright =
+
+#
+# Any additional imports go here
+#
+FILES_nlm_Ximports = \
+	@$(APR)/aprlib.imp \
+	@$(NOVI)/libc.imp \
+	$(EOLIST)
+ 
+#   
+# Any symbols exported to here
+#
+FILES_nlm_exports = \
+	$(EOLIST)
+
+#   
+# These are the OBJ files needed to create the LIB target above.
+# Paths must all use the '/' character
+#
+FILES_lib_objs = \
+	$(EOLIST)
+
+#
+# implement targets and dependancies (leave this section alone)
+#
+
+libs :: $(OBJDIR) $(TARGET_lib)
+
+nlms :: libs $(TARGET_nlm)
+
+#
+# Updated this target to create necessary directories and copy files to the 
+# correct place.  (See $(APR_WORK)/build/NWGNUhead.inc for examples)
+#
+install :: nlms FORCE
+
+#
+# Any specialized rules here
+#
+
+#
+# Include the 'tail' makefile that has targets that depend on variables defined
+# in this makefile
+#
+
+include $(APRBUILD)/NWGNUtail.inc
+
diff --git a/test/memcachedmock.c b/test/memcachedmock.c
new file mode 100644
index 00000000..b6c2e6fc
--- /dev/null
+++ b/test/memcachedmock.c
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "apr_network_io.h"
+#include "apr_pools.h"
+#include "apr_pools.h"
+#include "testmemcache.h"
+
+#define MOCK_REPLY "VERSION 1.5.22\r\n"
+
+int main(int argc, char *argv[])
+{
+    apr_pool_t *p;
+    apr_sockaddr_t *sa;
+    apr_socket_t *server;
+    apr_socket_t *server_connection;
+    apr_status_t rv;
+    apr_size_t length;
+    int i;
+
+    apr_initialize();
+    atexit(apr_terminate);
+    apr_pool_create(&p, NULL);
+
+    apr_sockaddr_info_get(&sa, MOCK_HOST, APR_UNSPEC, MOCK_PORT, 0, p);
+
+    apr_socket_create(&server, sa->family, SOCK_STREAM, 0, p);
+
+    apr_socket_opt_set(server, APR_SO_REUSEADDR, 1);
+
+    apr_socket_timeout_set(server, 0);
+
+    apr_socket_bind(server, sa);
+
+    apr_socket_listen(server, 5);
+
+    /* Do spin instead of a proper poll for sake of simplicity */
+    for (i = 0; i < 4; i++) {
+
+        rv = apr_socket_accept(&server_connection, server, p);
+        if (rv == APR_SUCCESS) {
+            break;
+        }
+
+        apr_sleep(apr_time_from_sec(1));
+    }
+
+    length = strlen(MOCK_REPLY);
+    apr_socket_send(server_connection, MOCK_REPLY, &length);
+
+    apr_socket_close(server_connection);
+
+    /* Do spin instead of a proper poll for sake of simplicity */
+    for (i = 0; i < 4; i++) {
+
+        rv = apr_socket_accept(&server_connection, server, p);
+        if (rv == APR_SUCCESS) {
+            break;
+        }
+
+        apr_sleep(apr_time_from_sec(1));
+    }
+
+    length = strlen(MOCK_REPLY);
+    apr_socket_send(server_connection, MOCK_REPLY, &length);
+
+    apr_socket_close(server_connection);
+
+    exit(0);
+}
diff --git a/test/testmemcache.c b/test/testmemcache.c
index c65381e3..5b498fea 100644
--- a/test/testmemcache.c
+++ b/test/testmemcache.c
@@ -22,9 +22,12 @@
 #include "apr_hash.h"
 #include "apr_memcache.h"
 #include "apr_network_io.h"
+#include "apr_thread_proc.h"
+#include "apr_signal.h"
+#include "testmemcache.h"
 
 #if APR_HAVE_STDLIB_H
-#include <stdlib.h>		/* for exit() */
+#include <stdlib.h>             /* for exit() */
 #endif
 
 #define HOST "localhost"
@@ -57,7 +60,7 @@ const char txt[] =
  * this datatype is for our custom server determination function. this might
  * be useful if you don't want to rely on simply hashing keys to determine
  * where a key belongs, but instead want to write something fancy, or use some
- * other kind of configuration data, i.e. a hash plus some data about a 
+ * other kind of configuration data, i.e. a hash plus some data about a
  * namespace, or whatever. see my_server_func, and test_memcache_user_funcs
  * for the examples.
  */
@@ -67,7 +70,7 @@ typedef struct {
 } my_hash_server_baton;
 
 
-/* this could do something fancy and return some hash result. 
+/* this could do something fancy and return some hash result.
  * for simplicity, just return the same value, so we can test it later on.
  * if you wanted to use some external hashing library or functions for
  * consistent hashing, for example, this would be a good place to do it.
@@ -82,10 +85,10 @@ static apr_uint32_t my_hash_func(void *baton, const char *data,
 /*
  * a fancy function to determine which server to use given some kind of data
  * and a hash value. this example actually ignores the hash value itself
- * and pulls some number from the *baton, which is a struct that has some 
+ * and pulls some number from the *baton, which is a struct that has some
  * kind of meaningful stuff in it.
  */
-static apr_memcache_server_t *my_server_func(void *baton, 
+static apr_memcache_server_t *my_server_func(void *baton,
                                              apr_memcache_t *mc,
                                              const apr_uint32_t hash)
 {
@@ -94,7 +97,7 @@ static apr_memcache_server_t *my_server_func(void *baton,
 
   if(mc->ntotal == 0) {
     return NULL;
-  } 
+  }
 
   if(mc->ntotal < mhsb->which_server) {
     return NULL;
@@ -112,8 +115,8 @@ static int randval(apr_uint32_t high)
     double d = 0;
 
     if (firsttime == 0) {
-	srand((unsigned) (getpid()));
-	firsttime = 1;
+        srand((unsigned) (getpid()));
+        firsttime = 1;
     }
 
     d = (double) rand() / ((double) RAND_MAX + 1);
@@ -139,40 +142,40 @@ static void test_memcache_create(abts_case * tc, void *data)
 
   rv = apr_memcache_create(pool, max_servers, 0, &memcache);
   ABTS_ASSERT(tc, "memcache create failed", rv == APR_SUCCESS);
-  
+
   for (i = 1; i <= max_servers; i++) {
     apr_port_t port;
-    
+
     port = PORT + i;
     rv =
       apr_memcache_server_create(pool, HOST, PORT + i, 0, 1, 1, 60, &server);
     ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
-    
+
     rv = apr_memcache_add_server(memcache, server);
     ABTS_ASSERT(tc, "server add failed", rv == APR_SUCCESS);
-    
+
     s = apr_memcache_find_server(memcache, HOST, port);
     ABTS_PTR_EQUAL(tc, server, s);
-    
+
     rv = apr_memcache_disable_server(memcache, s);
     ABTS_ASSERT(tc, "server disable failed", rv == APR_SUCCESS);
-    
+
     rv = apr_memcache_enable_server(memcache, s);
     ABTS_ASSERT(tc, "server enable failed", rv == APR_SUCCESS);
-    
+
     hash = apr_memcache_hash(memcache, prefix, strlen(prefix));
     ABTS_ASSERT(tc, "hash failed", hash > 0);
-    
+
     s = apr_memcache_find_server_hash(memcache, hash);
     ABTS_PTR_NOTNULL(tc, s);
   }
 
   rv = apr_memcache_server_create(pool, HOST, PORT, 0, 1, 1, 60, &server);
   ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_add_server(memcache, server);
   ABTS_ASSERT(tc, "server add should have failed", rv != APR_SUCCESS);
-  
+
 }
 
 /* install our own custom hashing and server selection routines. */
@@ -180,13 +183,13 @@ static void test_memcache_create(abts_case * tc, void *data)
 static int create_test_hash(apr_pool_t *p, apr_hash_t *h)
 {
   int i;
-  
+
   for (i = 0; i < TDATA_SIZE; i++) {
     char *k, *v;
-    
+
     k = apr_pstrcat(p, prefix, apr_itoa(p, i), NULL);
     v = apr_pstrndup(p, txt, randval((apr_uint32_t)strlen(txt)));
-    
+
     apr_hash_set(h, k, APR_HASH_KEY_STRING, v);
   }
 
@@ -202,13 +205,13 @@ static void test_memcache_user_funcs(abts_case * tc, void *data)
   apr_uint32_t max_servers = 10;
   apr_uint32_t hres;
   apr_uint32_t i;
-  my_hash_server_baton *baton = 
+  my_hash_server_baton *baton =
     apr_pcalloc(pool, sizeof(my_hash_server_baton));
 
   rv = apr_memcache_create(pool, max_servers, 0, &memcache);
   ABTS_ASSERT(tc, "memcache create failed", rv == APR_SUCCESS);
 
-  /* as noted above, install our custom hash function, and call 
+  /* as noted above, install our custom hash function, and call
    * apr_memcache_hash. the return value should be our predefined number,
    * and our function just ignores the other args, for simplicity.
    */
@@ -216,24 +219,24 @@ static void test_memcache_user_funcs(abts_case * tc, void *data)
 
   hres = apr_memcache_hash(memcache, "whatever", sizeof("whatever") - 1);
   ABTS_INT_EQUAL(tc, HASH_FUNC_RESULT, hres);
-  
+
   /* add some servers */
   for(i = 1; i <= 10; i++) {
     apr_memcache_server_t *ms;
 
     rv = apr_memcache_server_create(pool, HOST, i, 0, 1, 1, 60, &ms);
     ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
-    
+
     rv = apr_memcache_add_server(memcache, ms);
     ABTS_ASSERT(tc, "server add failed", rv == APR_SUCCESS);
   }
 
-  /* 
-   * set 'which_server' in our server_baton to find the third server 
+  /*
+   * set 'which_server' in our server_baton to find the third server
    * which should have the same port.
    */
   baton->which_server = 3;
-  memcache->server_func = my_server_func; 
+  memcache->server_func = my_server_func;
   memcache->server_baton = baton;
   found = apr_memcache_find_server_hash(memcache, 0);
   ABTS_ASSERT(tc, "wrong server found", found->port == baton->which_server);
@@ -266,11 +269,11 @@ static void test_memcache_meta(abts_case * tc, void *data)
 
     ABTS_STR_NEQUAL(tc, stats->version, result, 5);
 
-    /* 
+    /*
      * no way to know exactly what will be in most of these, so
      * just make sure there is something.
      */
-    
+
     ABTS_ASSERT(tc, "pid", stats->pid >= 0);
     ABTS_ASSERT(tc, "time", stats->time >= 0);
     /* ABTS_ASSERT(tc, "pointer_size", stats->pointer_size >= 0); */
@@ -283,7 +286,7 @@ static void test_memcache_meta(abts_case * tc, void *data)
 
     ABTS_ASSERT(tc, "curr_connections", stats->curr_connections >= 0);
     ABTS_ASSERT(tc, "total_connections", stats->total_connections >= 0);
-    ABTS_ASSERT(tc, "connection_structures", 
+    ABTS_ASSERT(tc, "connection_structures",
                 stats->connection_structures >= 0);
 
     ABTS_ASSERT(tc, "cmd_get", stats->cmd_get >= 0);
@@ -315,13 +318,13 @@ static void test_memcache_addreplace(abts_case * tc, void *data)
 
   rv = apr_memcache_create(pool, 1, 0, &memcache);
   ABTS_ASSERT(tc, "memcache create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_server_create(pool, HOST, PORT, 0, 1, 1, 60, &server);
   ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_add_server(memcache, server);
   ABTS_ASSERT(tc, "server add failed", rv == APR_SUCCESS);
-  
+
   tdata = apr_hash_make(p);
   create_test_hash(pool, tdata);
 
@@ -336,7 +339,7 @@ static void test_memcache_addreplace(abts_case * tc, void *data)
     /* doesn't exist yet, fail */
     rv = apr_memcache_replace(memcache, key, v, strlen(v) - 1, 0, 27);
     ABTS_ASSERT(tc, "replace should have failed", rv != APR_SUCCESS);
-    
+
     /* doesn't exist yet, succeed */
     rv = apr_memcache_add(memcache, key, v, strlen(v), 0, 27);
     ABTS_ASSERT(tc, "add failed", rv == APR_SUCCESS);
@@ -374,16 +377,16 @@ static void test_memcache_incrdecr(abts_case * tc, void *data)
 
   rv = apr_memcache_create(pool, 1, 0, &memcache);
   ABTS_ASSERT(tc, "memcache create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_server_create(pool, HOST, PORT, 0, 1, 1, 60, &server);
   ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_add_server(memcache, server);
   ABTS_ASSERT(tc, "server add failed", rv == APR_SUCCESS);
 
   rv = apr_memcache_set(memcache, prefix, "271", sizeof("271") - 1, 0, 27);
   ABTS_ASSERT(tc, "set failed", rv == APR_SUCCESS);
-  
+
   for( i = 1; i <= TDATA_SIZE; i++) {
     apr_uint32_t expect;
 
@@ -426,16 +429,16 @@ static void test_memcache_multiget(abts_case * tc, void *data)
 
   rv = apr_memcache_create(pool, 1, 0, &memcache);
   ABTS_ASSERT(tc, "memcache create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_server_create(pool, HOST, PORT, 0, 1, 1, 60, &server);
   ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
-  
+
   rv = apr_memcache_add_server(memcache, server);
   ABTS_ASSERT(tc, "server add failed", rv == APR_SUCCESS);
-  
+
   values = apr_hash_make(p);
   tdata = apr_hash_make(p);
-  
+
   create_test_hash(pool, tdata);
 
   for (hi = apr_hash_first(p, tdata); hi; hi = apr_hash_next(hi)) {
@@ -449,34 +452,34 @@ static void test_memcache_multiget(abts_case * tc, void *data)
     rv = apr_memcache_set(memcache, key, v, strlen(v), 0, 27);
     ABTS_ASSERT(tc, "set failed", rv == APR_SUCCESS);
   }
-  
+
   apr_pool_create(&tmppool, pool);
   for (i = 0; i < TDATA_SET; i++)
     apr_memcache_add_multget_key(pool,
                                  apr_pstrcat(pool, prefix,
                                              apr_itoa(pool, i), NULL),
                                  &values);
-    
+
   rv = apr_memcache_multgetp(memcache,
                              tmppool,
                              pool,
                              values);
-  
+
   ABTS_ASSERT(tc, "multgetp failed", rv == APR_SUCCESS);
   ABTS_ASSERT(tc, "multgetp returned too few results",
               apr_hash_count(values) == TDATA_SET);
-  
+
   for (hi = apr_hash_first(p, tdata); hi; hi = apr_hash_next(hi)) {
     const void *k;
     const char *key;
-    
+
     apr_hash_this(hi, &k, NULL, NULL);
     key = k;
 
     rv = apr_memcache_delete(memcache, key, 0);
     ABTS_ASSERT(tc, "delete failed", rv == APR_SUCCESS);
   }
-  
+
 }
 
 /* test setting and getting */
@@ -506,17 +509,17 @@ static void test_memcache_setget(abts_case * tc, void *data)
     create_test_hash(pool, tdata);
 
     for (hi = apr_hash_first(p, tdata); hi; hi = apr_hash_next(hi)) {
-	const void *k;
-	void *v;
+        const void *k;
+        void *v;
         const char *key;
 
-	apr_hash_this(hi, &k, NULL, &v);
+        apr_hash_this(hi, &k, NULL, &v);
         key = k;
 
-	rv = apr_memcache_set(memcache, key, v, strlen(v), 0, 27);
-	ABTS_ASSERT(tc, "set failed", rv == APR_SUCCESS);
-	rv = apr_memcache_getp(memcache, pool, key, &result, &len, NULL);
-	ABTS_ASSERT(tc, "get failed", rv == APR_SUCCESS);
+        rv = apr_memcache_set(memcache, key, v, strlen(v), 0, 27);
+        ABTS_ASSERT(tc, "set failed", rv == APR_SUCCESS);
+        rv = apr_memcache_getp(memcache, pool, key, &result, &len, NULL);
+        ABTS_ASSERT(tc, "get failed", rv == APR_SUCCESS);
     }
 
     rv = apr_memcache_getp(memcache, pool, "nothere3423", &result, &len, NULL);
@@ -524,19 +527,19 @@ static void test_memcache_setget(abts_case * tc, void *data)
     ABTS_ASSERT(tc, "get should have failed", rv != APR_SUCCESS);
 
     for (hi = apr_hash_first(p, tdata); hi; hi = apr_hash_next(hi)) {
-	const void *k;
-	const char *key;
+        const void *k;
+        const char *key;
 
-	apr_hash_this(hi, &k, NULL, NULL);
-	key = k;
+        apr_hash_this(hi, &k, NULL, NULL);
+        key = k;
 
-	rv = apr_memcache_delete(memcache, key, 0);
-	ABTS_ASSERT(tc, "delete failed", rv == APR_SUCCESS);
+        rv = apr_memcache_delete(memcache, key, 0);
+        ABTS_ASSERT(tc, "delete failed", rv == APR_SUCCESS);
     }
 }
 
-/* use apr_socket stuff to see if there is in fact a memcached server 
- * running on PORT. 
+/* use apr_socket stuff to see if there is in fact a memcached server
+ * running on PORT.
  */
 static apr_status_t check_mc(void)
 {
@@ -599,11 +602,80 @@ static apr_status_t check_mc(void)
   return rv;
 }
 
+static void test_connection_validation(abts_case *tc, void *data)
+{
+    apr_status_t rv;
+    apr_memcache_t *memcache;
+    apr_memcache_server_t *memserver;
+    char *result;
+    apr_procattr_t *procattr;
+    apr_proc_t proc;
+    const char *args[2];
+    int exitcode;
+    apr_exit_why_e why;
+#ifdef SIGPIPE
+    /*
+     * If SIGPIPE is present ignore it as we will write to a closed socket.
+     * Otherwise we would be terminated by the default handler for SIGPIPE.
+     */
+    apr_sigfunc_t *old_action;
+
+    old_action = apr_signal(SIGPIPE, SIG_IGN);
+#endif
+
+    rv = apr_procattr_create(&procattr, p);
+    ABTS_ASSERT(tc, "Couldn't create procattr", rv == APR_SUCCESS);
+
+    rv = apr_procattr_io_set(procattr, APR_NO_PIPE, APR_NO_PIPE,
+            APR_NO_PIPE);
+    ABTS_ASSERT(tc, "Couldn't set io in procattr", rv == APR_SUCCESS);
+
+    rv = apr_procattr_error_check_set(procattr, 1);
+    ABTS_ASSERT(tc, "Couldn't set error check in procattr", rv == APR_SUCCESS);
+
+    rv = apr_procattr_cmdtype_set(procattr, APR_PROGRAM_ENV);
+    ABTS_ASSERT(tc, "Couldn't set copy environment", rv == APR_SUCCESS);
+
+    args[0] = "memcachedmock" EXTENSION;
+    args[1] = NULL;
+    rv = apr_proc_create(&proc, TESTBINPATH "memcachedmock" EXTENSION, args, NULL,
+                         procattr, p);
+    ABTS_ASSERT(tc, "Couldn't launch program", rv == APR_SUCCESS);
+
+    /* Wait for the mock memcached to start */
+    apr_sleep(apr_time_from_sec(2));
+
+    rv = apr_memcache_create(p, 1, 0, &memcache);
+    ABTS_ASSERT(tc, "memcache create failed", rv == APR_SUCCESS);
+
+    rv = apr_memcache_server_create(p, MOCK_HOST, MOCK_PORT, 0, 1, 1, 60000, &memserver);
+    ABTS_ASSERT(tc, "server create failed", rv == APR_SUCCESS);
+
+    rv = apr_memcache_add_server(memcache, memserver);
+    ABTS_ASSERT(tc, "server add failed", rv == APR_SUCCESS);
+
+    rv = apr_memcache_version(memserver, p, &result);
+    ABTS_ASSERT(tc, "Couldn't get initial version", rv == APR_SUCCESS);
+
+    /* Wait for the mock memcached to shutdown the socket */
+    apr_sleep(apr_time_from_sec(1));
+
+    rv = apr_memcache_version(memserver, p, &result);
+    ABTS_ASSERT(tc, "Couldn't get version after connection shutdown", rv == APR_SUCCESS);
+
+#ifdef SIGPIPE
+    /* Restore old SIGPIPE handler */
+    apr_signal(SIGPIPE, old_action);
+#endif
+
+    apr_proc_wait(&proc, &exitcode, &why, APR_WAIT);
+}
+
 abts_suite *testmemcache(abts_suite * suite)
 {
     apr_status_t rv;
     suite = ADD_SUITE(suite);
-    /* check for a running memcached on the typical port before 
+    /* check for a running memcached on the typical port before
      * trying to run the tests. succeed if we don't find one.
      */
     rv = check_mc();
@@ -621,6 +693,7 @@ abts_suite *testmemcache(abts_suite * suite)
                          "on %s:%d.  Skipping apr_memcache tests...",
                          rv, HOST, PORT);
     }
+    abts_run_test(suite, test_connection_validation, NULL);
 
     return suite;
 }
diff --git a/test/testmemcache.h b/test/testmemcache.h
new file mode 100644
index 00000000..e7f7066e
--- /dev/null
+++ b/test/testmemcache.h
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TESTMEMCACHE_H
+#define TESTMEMCACHE_H
+
+#define MOCK_HOST "localhost"
+#define MOCK_PORT 11231
+
+#endif
+
diff --git a/test/testutil.h b/test/testutil.h
index c6e23887..2f557070 100644
--- a/test/testutil.h
+++ b/test/testutil.h
@@ -15,12 +15,26 @@
  */
 
 #include "apr_pools.h"
+#include "apr_general.h"
 #include "abts.h"
 
 #ifndef APR_TEST_UTIL
 #define APR_TEST_UTIL
 
-/* XXX FIXME */
+/* XXX: FIXME - these all should become much more utilitarian 
+ * and part of apr, itself
+ */
+
+#ifdef WIN32
+#ifdef BINPATH
+#define TESTBINPATH APR_STRINGIFY(BINPATH) "/"
+#else
+#define TESTBINPATH ""
+#endif
+#else
+#define TESTBINPATH "./"
+#endif
+
 #ifdef WIN32
 #define EXTENSION ".exe"
 #elif NETWARE