Write a symptom for checking memcache connections

This makes it easier for operators to troubleshoot connection issues to
Memcached.

Related-Bug: 1332058

Change-Id: I6e67363822480314b93608bb1eae3514f1480f6d

3 files changed, 88 insertions, 0 deletions

diff --git a/keystone/cmd/doctor/caching.py b/keystone/cmd/doctor/caching.py
index dc6997f01..c0e959020 100644
--- a/keystone/cmd/doctor/caching.py
+++ b/keystone/cmd/doctor/caching.py
@@ -10,6 +10,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from keystone.common import cache
 import keystone.conf
 
 
@@ -33,3 +34,27 @@ def symptom_caching_enabled_without_a_backend():
     point to a real caching backend like memcached.
     """
     return CONF.cache.enabled and CONF.cache.backend == 'dogpile.cache.null'
+
+
+def symptom_connection_to_memcached():
+    """Memcached isn't reachable.
+
+    Caching is enabled and the `keystone.conf [cache] backend` option is
+    configured but one or more Memcached servers are not reachable or marked
+    as dead. Please ensure `keystone.conf [cache] memcache_servers` is
+    configured properly.
+    """
+    memcached_drivers = [
+        'dogpile.cache.memcached',
+        'oslo_cache.memcache_pool'
+    ]
+    if CONF.cache.enabled and CONF.cache.backend in memcached_drivers:
+        cache.configure_cache()
+        cache_stats = cache.CACHE_REGION.actual_backend.client.get_stats()
+        memcached_server_count = len(CONF.cache.memcache_servers)
+        if len(cache_stats) != memcached_server_count:
+            return True
+        else:
+            return False
+    else:
+        return False
diff --git a/keystone/tests/unit/test_cli.py b/keystone/tests/unit/test_cli.py
index 572660a92..e2c42ec8a 100644
--- a/keystone/tests/unit/test_cli.py
+++ b/keystone/tests/unit/test_cli.py
@@ -956,6 +956,63 @@ class CachingDoctorTests(unit.TestCase):
                                    backend='dogpile.cache.memory')
         self.assertFalse(caching.symptom_caching_enabled_without_a_backend())
 
+    @mock.patch('keystone.cmd.doctor.caching.cache.CACHE_REGION')
+    def test_symptom_connection_to_memcached(self, cache_mock):
+        self.config_fixture.config(group='cache', enabled=True)
+        self.config_fixture.config(
+            group='cache',
+            memcache_servers=['alpha.com:11211', 'beta.com:11211']
+        )
+        self.config_fixture.config(
+            group='cache', backend='dogpile.cache.memcached'
+        )
+
+        # No symptom detected: Caching driver can connect to both memcached
+        # servers
+        cache_mock.actual_backend.client.get_stats.return_value = (
+            [('alpha.com', {}), ('beta.com', {})]
+        )
+        self.assertFalse(caching.symptom_connection_to_memcached())
+
+        # Symptom detected: Caching driver can't connect to either memcached
+        # server
+        cache_mock.actual_backend.client.get_stats.return_value = []
+        self.assertTrue(caching.symptom_connection_to_memcached())
+
+        # Symptom detected: Caching driver can't connect to one memcached
+        # server
+        cache_mock.actual_backend.client.get_stats.return_value = [
+            ('alpha.com', {})
+        ]
+        self.assertTrue(caching.symptom_connection_to_memcached())
+
+        self.config_fixture.config(
+            group='cache',
+            memcache_servers=['alpha.com:11211', 'beta.com:11211']
+        )
+        self.config_fixture.config(
+            group='cache', backend='oslo_cache.memcache_pool'
+        )
+
+        # No symptom detected: Caching driver can connect to both memcached
+        # servers
+        cache_mock.actual_backend.client.get_stats.return_value = (
+            [('alpha.com', {}), ('beta.com', {})]
+        )
+        self.assertFalse(caching.symptom_connection_to_memcached())
+
+        # Symptom detected: Caching driver can't connect to either memcached
+        # server
+        cache_mock.actual_backend.client.get_stats.return_value = []
+        self.assertTrue(caching.symptom_connection_to_memcached())
+
+        # Symptom detected: Caching driver can't connect to one memcached
+        # server
+        cache_mock.actual_backend.client.get_stats.return_value = [
+            ('alpha.com', {})
+        ]
+        self.assertTrue(caching.symptom_connection_to_memcached())
+
 
 class CredentialDoctorTests(unit.TestCase):
 
diff --git a/releasenotes/notes/bug-1332058-f25e2de40411b711.yaml b/releasenotes/notes/bug-1332058-f25e2de40411b711.yaml
new file mode 100644
index 000000000..892bb6956
--- /dev/null
+++ b/releasenotes/notes/bug-1332058-f25e2de40411b711.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    [`bug 1332058 <https://bugs.launchpad.net/keystone/+bug/1332058>`_]
+    ``keystone-manage doctor`` now checks that keystone can establish
+    connections to Memcached, if configured.