Added server_timeout_counter and behavior MEMCACHED_BEHAVIOR_SERVER_TIMEOUT_LIMIT.

This feature works as follows:
When a timeout occurs, retry MEMCACHED_BEHAVIOR_SERVER_TIMEOUT_LIMIT times before
putting the server in MEMCACHED_SERVER_STATE_IN_TIMEOUT state.

It makes sense to not fail from the 1st time but retry on the next calls before marking
the server as timeout.

13 files changed, 37 insertions, 8 deletions

diff --git a/libmemcached-1.0/defaults.h b/libmemcached-1.0/defaults.h
index 177e18cf..d440f8c8 100644
--- a/libmemcached-1.0/defaults.h
+++ b/libmemcached-1.0/defaults.h
@@ -52,5 +52,5 @@
 #define MEMCACHED_SERVER_FAILURE_LIMIT 5
 #define MEMCACHED_SERVER_FAILURE_RETRY_TIMEOUT 2
 #define MEMCACHED_SERVER_FAILURE_DEAD_TIMEOUT 0
-
+#define MEMCACHED_SERVER_TIMEOUT_LIMIT 0
 
diff --git a/libmemcached-1.0/struct/memcached.h b/libmemcached-1.0/struct/memcached.h
index 37df2751..a368bfe8 100644
--- a/libmemcached-1.0/struct/memcached.h
+++ b/libmemcached-1.0/struct/memcached.h
@@ -79,6 +79,7 @@ struct memcached_st {
   int32_t snd_timeout;
   int32_t rcv_timeout;
   uint32_t server_failure_limit;
+  uint32_t server_timeout_limit;
   uint32_t io_msg_watermark;
   uint32_t io_bytes_watermark;
   uint32_t io_key_prefetch;
diff --git a/libmemcached-1.0/struct/server.h b/libmemcached-1.0/struct/server.h
index 33bdc049..9337938d 100644
--- a/libmemcached-1.0/struct/server.h
+++ b/libmemcached-1.0/struct/server.h
@@ -77,6 +77,8 @@ struct memcached_server_st {
   uint32_t request_id;
   uint32_t server_failure_counter;
   uint64_t server_failure_counter_query_id;
+  uint32_t server_timeout_counter;
+  uint64_t server_timeout_counter_query_id;
   uint32_t weight;
   uint32_t version;
   enum memcached_server_state_t state;
diff --git a/libmemcached-1.0/types/behavior.h b/libmemcached-1.0/types/behavior.h
index d0af189a..a1bbfd22 100644
--- a/libmemcached-1.0/types/behavior.h
+++ b/libmemcached-1.0/types/behavior.h
@@ -76,6 +76,7 @@ enum memcached_behavior_t {
   MEMCACHED_BEHAVIOR_LOAD_FROM_FILE,
   MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS,
   MEMCACHED_BEHAVIOR_DEAD_TIMEOUT,
+  MEMCACHED_BEHAVIOR_SERVER_TIMEOUT_LIMIT,
   MEMCACHED_BEHAVIOR_MAX
 };
 
diff --git a/libmemcached/behavior.cc b/libmemcached/behavior.cc
index 081f8082..40fb5fb6 100644
--- a/libmemcached/behavior.cc
+++ b/libmemcached/behavior.cc
@@ -116,6 +116,10 @@ memcached_return_t memcached_behavior_set(memcached_st *shell,
     ptr->server_failure_limit= uint32_t(data);
     break;
 
+  case MEMCACHED_BEHAVIOR_SERVER_TIMEOUT_LIMIT:
+    ptr->server_timeout_limit= uint32_t(data);
+    break;
+
   case MEMCACHED_BEHAVIOR_BINARY_PROTOCOL:
     send_quit(ptr); // We need t shutdown all of the connections to make sure we do the correct protocol
     if (data)
@@ -374,6 +378,9 @@ uint64_t memcached_behavior_get(memcached_st *shell,
   case MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT:
     return ptr->server_failure_limit;
 
+  case MEMCACHED_BEHAVIOR_SERVER_TIMEOUT_LIMIT:
+    return ptr->server_timeout_limit;
+
   case MEMCACHED_BEHAVIOR_SORT_HOSTS:
     return ptr->flags.use_sort_hosts;
 
diff --git a/libmemcached/connect.cc b/libmemcached/connect.cc
index 3c82f641..52cd293a 100644
--- a/libmemcached/connect.cc
+++ b/libmemcached/connect.cc
@@ -713,6 +713,7 @@ static memcached_return_t backoff_handling(memcached_instance_st* server, bool&
     if (_gettime_success and server->next_retry < curr_time.tv_sec)
     {
       server->state= MEMCACHED_SERVER_STATE_NEW;
+      server->server_timeout_counter= 0;
     }
     else
     {
diff --git a/libmemcached/instance.cc b/libmemcached/instance.cc
index 82621f7a..53f74f79 100644
--- a/libmemcached/instance.cc
+++ b/libmemcached/instance.cc
@@ -54,6 +54,8 @@ static inline void _server_init(memcached_instance_st* self, Memcached *root,
   self->request_id= 0;
   self->server_failure_counter= 0;
   self->server_failure_counter_query_id= 0;
+  self->server_timeout_counter= 0;
+  self->server_timeout_counter_query_id= 0;
   self->weight= weight ? weight : 1; // 1 is the default weight value
   self->io_wait_count.read= 0;
   self->io_wait_count.write= 0;
diff --git a/libmemcached/instance.hpp b/libmemcached/instance.hpp
index c76dc7ee..73d74a66 100644
--- a/libmemcached/instance.hpp
+++ b/libmemcached/instance.hpp
@@ -73,6 +73,7 @@ struct memcached_instance_st {
   void mark_server_as_clean()
   {
     server_failure_counter= 0;
+    server_timeout_counter= 0;
     next_retry= 0;
   }
 
@@ -147,6 +148,8 @@ struct memcached_instance_st {
   uint32_t request_id;
   uint32_t server_failure_counter;
   uint64_t server_failure_counter_query_id;
+  uint32_t server_timeout_counter;
+  uint32_t server_timeout_counter_query_id;
   uint32_t weight;
   uint32_t version;
   enum memcached_server_state_t state;
diff --git a/libmemcached/memcached.cc b/libmemcached/memcached.cc
index a3c6a679..337f918d 100644
--- a/libmemcached/memcached.cc
+++ b/libmemcached/memcached.cc
@@ -87,6 +87,7 @@ static inline bool _memcached_init(Memcached *self)
   self->snd_timeout= 0;
   self->rcv_timeout= 0;
   self->server_failure_limit= MEMCACHED_SERVER_FAILURE_LIMIT;
+  self->server_timeout_limit= MEMCACHED_SERVER_TIMEOUT_LIMIT;
   self->query_id= 1; // 0 is considered invalid
 
   /* TODO, Document why we picked these defaults */
@@ -362,6 +363,7 @@ memcached_st *memcached_clone(memcached_st *clone, const memcached_st *source)
   new_clone->get_key_failure= source->get_key_failure;
   new_clone->delete_trigger= source->delete_trigger;
   new_clone->server_failure_limit= source->server_failure_limit;
+  new_clone->server_timeout_limit= source->server_timeout_limit;
   new_clone->io_msg_watermark= source->io_msg_watermark;
   new_clone->io_bytes_watermark= source->io_bytes_watermark;
   new_clone->io_key_prefetch= source->io_key_prefetch;
diff --git a/libmemcached/quit.cc b/libmemcached/quit.cc
index 04b77cea..5d17b313 100644
--- a/libmemcached/quit.cc
+++ b/libmemcached/quit.cc
@@ -93,6 +93,7 @@ namespace {
      * sent to the server.
      */
     instance->server_failure_counter= 0;
+    instance->server_timeout_counter= 0;
   }
 }
 
diff --git a/libmemcached/server.cc b/libmemcached/server.cc
index c4747ddc..e2837c7d 100644
--- a/libmemcached/server.cc
+++ b/libmemcached/server.cc
@@ -54,6 +54,8 @@ static inline void _server_init(memcached_server_st *self, Memcached *root,
   self->request_id= 0;
   self->server_failure_counter= 0;
   self->server_failure_counter_query_id= 0;
+  self->server_timeout_counter= 0;
+  self->server_timeout_counter_query_id= 0;
   self->weight= weight ? weight : 1; // 1 is the default weight value
   self->io_wait_count.read= 0;
   self->io_wait_count.write= 0;
diff --git a/libmemcached/server.hpp b/libmemcached/server.hpp
index 708e6460..6ea0b7f2 100644
--- a/libmemcached/server.hpp
+++ b/libmemcached/server.hpp
@@ -76,7 +76,13 @@ static inline void memcached_mark_server_for_timeout(memcached_instance_st* serv
 {
   if (server->state != MEMCACHED_SERVER_STATE_IN_TIMEOUT)
   {
-    if (server->root->retry_timeout != 0)
+    if (server->server_timeout_counter_query_id != server->root->query_id)
+    {
+      server->server_timeout_counter++;
+      server->server_timeout_counter_query_id= server->root->query_id;
+    }
+
+    if (server->server_timeout_counter >= server->root->server_timeout_limit)
     {
       struct timeval next_time;
       if (gettimeofday(&next_time, NULL) == 0)
@@ -89,12 +95,12 @@ static inline void memcached_mark_server_for_timeout(memcached_instance_st* serv
       }
 
       server->state= MEMCACHED_SERVER_STATE_IN_TIMEOUT;
+      if (server->server_failure_counter_query_id != server->root->query_id)
+      {
+        server->server_failure_counter++;
+        server->server_failure_counter_query_id= server->root->query_id;
+      }
+      set_last_disconnected_host(server);
     }
-    if (server->server_failure_counter_query_id != server->root->query_id)
-    {
-      server->server_failure_counter++;
-      server->server_failure_counter_query_id= server->root->query_id;
-    }
-    set_last_disconnected_host(server);
   }
 }
diff --git a/tests/libmemcached-1.0/mem_functions.cc b/tests/libmemcached-1.0/mem_functions.cc
index 365839d5..321a35db 100644
--- a/tests/libmemcached-1.0/mem_functions.cc
+++ b/tests/libmemcached-1.0/mem_functions.cc
@@ -412,6 +412,7 @@ test_return_t clone_test(memcached_st *memc)
     test_true(memc_clone->retry_timeout == memc->retry_timeout);
     test_true(memc_clone->send_size == memc->send_size);
     test_true(memc_clone->server_failure_limit == memc->server_failure_limit);
+    test_true(memc_clone->server_timeout_limit == memc->server_timeout_limit);
     test_true(memc_clone->snd_timeout == memc->snd_timeout);
     test_true(memc_clone->user_data == memc->user_data);