diff options
| -rw-r--r-- | docs/apidoc/pymemcache.client.rst | 46 | ||||
| -rw-r--r-- | docs/apidoc/pymemcache.rst | 7 | ||||
| -rw-r--r-- | docs/apidoc/pymemcache.test.rst | 16 | ||||
| -rw-r--r-- | docs/getting_started.rst | 21 | ||||
| -rw-r--r-- | pymemcache/client/__init__.py | 3 | ||||
| -rw-r--r-- | pymemcache/client/base.py (renamed from pymemcache/client.py) | 65 | ||||
| -rw-r--r-- | pymemcache/client/hash.py | 303 | ||||
| -rw-r--r-- | pymemcache/client/murmur3.py | 51 | ||||
| -rw-r--r-- | pymemcache/client/rendezvous.py | 46 | ||||
| -rw-r--r-- | pymemcache/exceptions.py | 40 | ||||
| -rw-r--r-- | pymemcache/test/conftest.py | 14 | ||||
| -rw-r--r-- | pymemcache/test/test_client.py | 13 | ||||
| -rw-r--r-- | pymemcache/test/test_client_hash.py | 111 | ||||
| -rw-r--r-- | pymemcache/test/test_integration.py | 40 | ||||
| -rw-r--r-- | pymemcache/test/test_rendezvous.py | 203 | ||||
| -rw-r--r-- | pymemcache/test/utils.py | 2 | ||||
| -rw-r--r-- | setup.py | 6 | ||||
| -rw-r--r-- | tox.ini | 4 |
18 files changed, 908 insertions, 83 deletions
diff --git a/docs/apidoc/pymemcache.client.rst b/docs/apidoc/pymemcache.client.rst new file mode 100644 index 0000000..0f2622b --- /dev/null +++ b/docs/apidoc/pymemcache.client.rst @@ -0,0 +1,46 @@ +pymemcache.client package +========================= + +Submodules +---------- + +pymemcache.client.base module +----------------------------- + +.. automodule:: pymemcache.client.base + :members: + :undoc-members: + :show-inheritance: + +pymemcache.client.hash module +----------------------------- + +.. automodule:: pymemcache.client.hash + :members: + :undoc-members: + :show-inheritance: + +pymemcache.client.murmur3 module +-------------------------------- + +.. automodule:: pymemcache.client.murmur3 + :members: + :undoc-members: + :show-inheritance: + +pymemcache.client.rendezvous module +----------------------------------- + +.. automodule:: pymemcache.client.rendezvous + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: pymemcache.client + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/apidoc/pymemcache.rst b/docs/apidoc/pymemcache.rst index 24a4fe8..8009023 100644 --- a/docs/apidoc/pymemcache.rst +++ b/docs/apidoc/pymemcache.rst @@ -6,15 +6,16 @@ Subpackages .. toctree:: + pymemcache.client pymemcache.test Submodules ---------- -pymemcache.client module ------------------------- +pymemcache.exceptions module +---------------------------- -.. automodule:: pymemcache.client +.. automodule:: pymemcache.exceptions :members: :undoc-members: :show-inheritance: diff --git a/docs/apidoc/pymemcache.test.rst b/docs/apidoc/pymemcache.test.rst index 8b1e17b..4b9f09d 100644 --- a/docs/apidoc/pymemcache.test.rst +++ b/docs/apidoc/pymemcache.test.rst @@ -28,6 +28,14 @@ pymemcache.test.test_client module :undoc-members: :show-inheritance: +pymemcache.test.test_client_hash module +--------------------------------------- + +.. automodule:: pymemcache.test.test_client_hash + :members: + :undoc-members: + :show-inheritance: + pymemcache.test.test_integration module --------------------------------------- @@ -36,6 +44,14 @@ pymemcache.test.test_integration module :undoc-members: :show-inheritance: +pymemcache.test.test_rendezvous module +-------------------------------------- + +.. automodule:: pymemcache.test.test_rendezvous + :members: + :undoc-members: + :show-inheritance: + pymemcache.test.test_utils module --------------------------------- diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 68125fc..10b66c1 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -7,12 +7,29 @@ Basic Usage .. code-block:: python - from pymemcache.client import Client + from pymemcache.client.base import Client client = Client(('localhost', 11211)) client.set('some_key', 'some_value') result = client.get('some_key') +Using a memcached cluster +------------------------- +This will use a consistent hashing algorithm to choose which server to +set/get the values from. It will also automatically rebalance depending +on if a server goes down. + +.. code-block:: python + + from pymemcache.client.hash import HashClient + + client = HashClient([ + ('127.0.0.1', 11211), + ('127.0.0.1', 11212) + ]) + client.set('some_key', 'some value') + result = client.get('some_key') + Serialization -------------- @@ -20,7 +37,7 @@ Serialization .. code-block:: python import json - from pymemcache.client import Client + from pymemcache.client.base import Client def json_serializer(key, value): if type(value) == str: diff --git a/pymemcache/client/__init__.py b/pymemcache/client/__init__.py new file mode 100644 index 0000000..779283a --- /dev/null +++ b/pymemcache/client/__init__.py @@ -0,0 +1,3 @@ +# API Backwards compatibility +from pymemcache.client.base import Client # noqa +from pymemcache.client.base import PooledClient # noqa diff --git a/pymemcache/client.py b/pymemcache/client/base.py index 9b5ae2c..f9d7973 100644 --- a/pymemcache/client.py +++ b/pymemcache/client/base.py @@ -20,6 +20,15 @@ import six from pymemcache import pool +from pymemcache.exceptions import ( + MemcacheClientError, + MemcacheUnknownCommandError, + MemcacheIllegalInputError, + MemcacheServerError, + MemcacheUnknownError, + MemcacheUnexpectedCloseError +) + RECV_SIZE = 4096 VALID_STORE_RESULTS = { @@ -56,51 +65,9 @@ STAT_TYPES = { b'slab_automove': lambda value: int(value) != 0, } - -class MemcacheError(Exception): - "Base exception class" - pass - - -class MemcacheClientError(MemcacheError): - """Raised when memcached fails to parse the arguments to a request, likely - due to a malformed key and/or value, a bug in this library, or a version - mismatch with memcached.""" - pass - - -class MemcacheUnknownCommandError(MemcacheClientError): - """Raised when memcached fails to parse a request, likely due to a bug in - this library or a version mismatch with memcached.""" - pass - - -class MemcacheIllegalInputError(MemcacheClientError): - """Raised when a key or value is not legal for Memcache (see the class docs - for Client for more details).""" - pass - - -class MemcacheServerError(MemcacheError): - """Raised when memcached reports a failure while processing a request, - likely due to a bug or transient issue in memcached.""" - pass - - -class MemcacheUnknownError(MemcacheError): - """Raised when this library receives a response from memcached that it - cannot parse, likely due to a bug in this library or a version mismatch - with memcached.""" - pass - - -class MemcacheUnexpectedCloseError(MemcacheServerError): - "Raised when the connection with memcached closes unexpectedly." - pass - - # Common helper functions. + def _check_key(key, key_prefix=b''): """Checks key and add key_prefix.""" if isinstance(key, six.text_type): @@ -120,8 +87,7 @@ class Client(object): """ A client for a single memcached server. - Keys and Values - ---------------- + *Keys and Values* Keys must have a __str__() method which should return a str with no more than 250 ASCII characters and no whitespace or control characters. Unicode @@ -131,7 +97,7 @@ class Client(object): Values must have a __str__() method to convert themselves to a byte string. Unicode objects can be a problem since str() on a Unicode object will attempt to encode it as ASCII (which will fail if the value contains - code points larger than U+127). You can fix this will a serializer or by + code points larger than U+127). You can fix this with a serializer or by just calling encode on the string (using UTF-8, for instance). If you intend to use anything but str as a value, it is a good idea to use @@ -139,8 +105,7 @@ class Client(object): already implemented serializers, including one that is compatible with the python-memcache library. - Serialization and Deserialization - ---------------------------------- + *Serialization and Deserialization* The constructor takes two optional functions, one for "serialization" of values, and one for "deserialization". The serialization function takes @@ -167,8 +132,7 @@ class Client(object): raise Exception("Unknown flags for value: {1}".format(flags)) - Error Handling - --------------- + *Error Handling* All of the methods in this class that talk to memcached can throw one of the following exceptions: @@ -656,7 +620,6 @@ class Client(object): while True: buf, line = _readline(self.sock, buf) self._raise_errors(line, name) - if line == b'END': return result elif line.startswith(b'VALUE'): diff --git a/pymemcache/client/hash.py b/pymemcache/client/hash.py new file mode 100644 index 0000000..393f6d0 --- /dev/null +++ b/pymemcache/client/hash.py @@ -0,0 +1,303 @@ +import socket +import time +import logging + +from pymemcache.client.base import Client, PooledClient, _check_key +from pymemcache.client.rendezvous import RendezvousHash + +logger = logging.getLogger(__name__) + + +class HashClient(object): + """ + A client for communicating with a cluster of memcached servers + """ + def __init__( + self, + servers, + hasher=None, + serializer=None, + deserializer=None, + connect_timeout=None, + timeout=None, + no_delay=False, + socket_module=socket, + key_prefix=b'', + max_pool_size=None, + lock_generator=None, + retry_attempts=2, + retry_timeout=1, + dead_timeout=60, + use_pooling=False, + ignore_exc=False, + ): + """ + Constructor. + + Args: + servers: list(tuple(hostname, port)) + hasher: optional class three functions ``get_node``, ``add_node``, + and ``remove_node`` + defaults to Rendezvous (HRW) hash. + + use_pooling: use py:class:`.PooledClient` as the default underlying + class. ``max_pool_size`` and ``lock_generator`` can + be used with this. default: False + + retry_attempts: Amount of times a client should be tried before it + is marked dead and removed from the pool. + retry_timeout (float): Time in seconds that should pass between retry + attempts. + dead_timeout (float): Time in seconds before attempting to add a node + back in the pool. + + Further arguments are interpreted as for :py:class:`.Client` + constructor. + + The default ``hasher`` is using a pure python implementation that can + be significantly improved performance wise by switching to a C based + version. We recommend using ``python-clandestined`` if having a C + dependency is acceptable. + """ + self.clients = {} + self.retry_attempts = retry_attempts + self.retry_timeout = retry_timeout + self.dead_timeout = dead_timeout + self.use_pooling = use_pooling + self.key_prefix = key_prefix + self.ignore_exc = ignore_exc + self._failed_clients = {} + self._dead_clients = {} + self._last_dead_check_time = time.time() + + if hasher is None: + self.hasher = RendezvousHash() + + self.default_kwargs = { + 'connect_timeout': connect_timeout, + 'timeout': timeout, + 'no_delay': no_delay, + 'socket_module': socket_module, + 'key_prefix': key_prefix, + 'serializer': serializer, + 'deserializer': deserializer, + } + + if use_pooling is True: + self.default_kwargs.update({ + 'max_pool_size': max_pool_size, + 'lock_generator': lock_generator + }) + + for server, port in servers: + self.add_server(server, port) + + def add_server(self, server, port): + key = '%s:%s' % (server, port) + + if self.use_pooling: + client = PooledClient( + (server, port), + **self.default_kwargs + ) + else: + client = Client((server, port)) + + self.clients[key] = client + self.hasher.add_node(key) + + def remove_server(self, server, port): + dead_time = time.time() + self._failed_clients.pop((server, port)) + self._dead_clients[(server, port)] = dead_time + key = '%s:%s' % (server, port) + self.hasher.remove_node(key) + + def _get_client(self, key): + _check_key(key, self.key_prefix) + if len(self._dead_clients) > 0: + current_time = time.time() + ldc = self._last_dead_check_time + # we have dead clients and we have reached the + # timeout retry + if current_time - ldc > self.dead_timeout: + for server, dead_time in self._dead_clients.items(): + if current_time - dead_time > self.dead_timeout: + logger.debug( + 'bringing server back into rotation %s', + server + ) + self.add_server(*server) + self._last_dead_check_time = current_time + + server = self.hasher.get_node(key) + client = self.clients[server] + return client + + def _safely_run_func(self, client, func, default_val, *args, **kwargs): + try: + if client.server in self._failed_clients: + # This server is currently failing, lets check if it is in + # retry or marked as dead + failed_metadata = self._failed_clients[client.server] + + # we haven't tried our max amount yet, if it has been enough + # time lets just retry using it + if failed_metadata['attempts'] < self.retry_attempts: + failed_time = failed_metadata['failed_time'] + if time.time() - failed_time > self.retry_timeout: + logger.debug( + 'retrying failed server: %s', client.server + ) + result = func(*args, **kwargs) + # we were successful, lets remove it from the failed + # clients + self._failed_clients.pop(client.server) + return result + return default_val + else: + # We've reached our max retry attempts, we need to mark + # the sever as dead + logger.debug('marking server as dead: %s', client.server) + self.remove_server(*client.server) + + result = func(*args, **kwargs) + return result + + # Connecting to the server fail, we should enter + # retry mode + except socket.error: + # This client has never failed, lets mark it for failure + if ( + client.server not in self._failed_clients and + self.retry_attempts > 0 + ): + self._failed_clients[client.server] = { + 'failed_time': time.time(), + 'attempts': 0, + } + # We aren't allowing any retries, we should mark the server as + # dead immediately + elif ( + client.server not in self._failed_clients and + self.retry_attempts < 0 + ): + self._failed_clients[client.server] = { + 'failed_time': time.time(), + 'attempts': 0, + } + logger.debug("marking server as dead %s" % client.server) + self.remove_server(*client.server) + # This client has failed previously, we need to update the metadata + # to reflect that we have attempted it again + else: + failed_metadata = self._failed_clients[client.server] + failed_metadata['attempts'] += 1 + failed_metadata['failed_time'] = time.time() + self._failed_clients[client.server] = failed_metadata + + # if we haven't enabled ignore_exc, don't move on gracefully, just + # raise the exception + if not self.ignore_exc: + raise + + return default_val + except: + # any exceptions that aren't socket.error we need to handle + # gracefully as well + if not self.ignore_exc: + raise + + return default_val + + def _run_cmd(self, cmd, key, default_val, *args, **kwargs): + client = self._get_client(key) + func = getattr(client, cmd) + args = list(args) + args.insert(0, key) + return self._safely_run_func( + client, func, default_val, *args, **kwargs + ) + + def set(self, key, *args, **kwargs): + return self._run_cmd('set', key, False, *args, **kwargs) + + def get(self, key, *args, **kwargs): + return self._run_cmd('get', key, None, *args, **kwargs) + + def incr(self, key, *args, **kwargs): + return self._run_cmd('incr', key, False, *args, **kwargs) + + def decr(self, key, *args, **kwargs): + return self._run_cmd('decr', key, False, *args, **kwargs) + + def set_many(self, values, *args, **kwargs): + client_batches = {} + for key, value in values.items(): + client = self._get_client(key) + if client.server not in client_batches: + client_batches[client.server] = {} + + client_batches[client.server][key] = value + + end = [] + + for server, values in client_batches.items(): + client = self.clients['%s:%s' % server] + new_args = list(args) + new_args.insert(0, values) + result = self._safely_run_func( + client, + client.set_many, False, *new_args, **kwargs + ) + end.append(result) + + return all(end) + + def get_many(self, keys, *args, **kwargs): + client_batches = {} + for key in keys: + client = self._get_client(key) + if client.server not in client_batches: + client_batches[client.server] = [] + + client_batches[client.server].append(key) + + end = {} + + for server, keys in client_batches.items(): + client = self.clients['%s:%s' % server] + new_args = list(args) + new_args.insert(0, keys) + result = self._safely_run_func( + client, + client.get_many, {}, *new_args, **kwargs + ) + end.update(result) + + return end + + def gets(self, key, *args, **kwargs): + return self._run_cmd('gets', key, None, *args, **kwargs) + + def add(self, key, *args, **kwargs): + return self._run_cmd('add', key, False, *args, **kwargs) + + def prepend(self, key, *args, **kwargs): + return self._run_cmd('prepend', key, False, *args, **kwargs) + + def append(self, key, *args, **kwargs): + return self._run_cmd('append', key, False, *args, **kwargs) + + def delete(self, key, *args, **kwargs): + return self._run_cmd('delete', key, False, *args, **kwargs) + + def cas(self, key, *args, **kwargs): + return self._run_cmd('cas', key, False, *args, **kwargs) + + def replace(self, key, *args, **kwargs): + return self._run_cmd('replace', key, False, *args, **kwargs) + + def flush_all(self): + for _, client in self.clients.items(): + self._safely_run_func(client, client.flush_all, False) diff --git a/pymemcache/client/murmur3.py b/pymemcache/client/murmur3.py new file mode 100644 index 0000000..787eeaf --- /dev/null +++ b/pymemcache/client/murmur3.py @@ -0,0 +1,51 @@ +def murmur3_32(data, seed=0): + """MurmurHash3 was written by Austin Appleby, and is placed in the + public domain. The author hereby disclaims copyright to this source + code.""" + + c1 = 0xcc9e2d51 + c2 = 0x1b873593 + + length = len(data) + h1 = seed + roundedEnd = (length & 0xfffffffc) # round down to 4 byte block + for i in range(0, roundedEnd, 4): + # little endian load order + k1 = (ord(data[i]) & 0xff) | ((ord(data[i + 1]) & 0xff) << 8) | \ + ((ord(data[i + 2]) & 0xff) << 16) | (ord(data[i + 3]) << 24) + k1 *= c1 + k1 = (k1 << 15) | ((k1 & 0xffffffff) >> 17) # ROTL32(k1,15) + k1 *= c2 + + h1 ^= k1 + h1 = (h1 << 13) | ((h1 & 0xffffffff) >> 19) # ROTL32(h1,13) + h1 = h1 * 5 + 0xe6546b64 + + # tail + k1 = 0 + + val = length & 0x03 + if val == 3: + k1 = (ord(data[roundedEnd + 2]) & 0xff) << 16 + # fallthrough + if val in [2, 3]: + k1 |= (ord(data[roundedEnd + 1]) & 0xff) << 8 + # fallthrough + if val in [1, 2, 3]: + k1 |= ord(data[roundedEnd]) & 0xff + k1 *= c1 + k1 = (k1 << 15) | ((k1 & 0xffffffff) >> 17) # ROTL32(k1,15) + k1 *= c2 + h1 ^= k1 + + # finalization + h1 ^= length + + # fmix(h1) + h1 ^= ((h1 & 0xffffffff) >> 16) + h1 *= 0x85ebca6b + h1 ^= ((h1 & 0xffffffff) >> 13) + h1 *= 0xc2b2ae35 + h1 ^= ((h1 & 0xffffffff) >> 16) + + return h1 & 0xffffffff diff --git a/pymemcache/client/rendezvous.py b/pymemcache/client/rendezvous.py new file mode 100644 index 0000000..32ecc2b --- /dev/null +++ b/pymemcache/client/rendezvous.py @@ -0,0 +1,46 @@ +from pymemcache.client.murmur3 import murmur3_32 + + +class RendezvousHash(object): + """ + Implements the Highest Random Weight (HRW) hashing algorithm most + commonly referred to as rendezvous hashing. + + Originally developed as part of python-clandestined. + + Copyright (c) 2014 Ernest W. Durbin III + """ + def __init__(self, nodes=None, seed=0, hash_function=murmur3_32): + """ + Constructor. + """ + self.nodes = [] + self.seed = seed + if nodes is not None: + self.nodes = nodes + self.hash_function = lambda x: hash_function(x, seed) + + def add_node(self, node): + if node not in self.nodes: + self.nodes.append(node) + + def remove_node(self, node): + if node in self.nodes: + self.nodes.remove(node) + else: + raise ValueError("No such node %s to remove" % (node)) + + def get_node(self, key): + high_score = -1 + winner = None + + for node in self.nodes: + score = self.hash_function( + "%s-%s" % (str(node), str(key))) + + if score > high_score: + (high_score, winner) = (score, node) + elif score == high_score: + (high_score, winner) = (score, max(str(node), str(winner))) + + return winner diff --git a/pymemcache/exceptions.py b/pymemcache/exceptions.py new file mode 100644 index 0000000..416fa0a --- /dev/null +++ b/pymemcache/exceptions.py @@ -0,0 +1,40 @@ +class MemcacheError(Exception): + "Base exception class" + pass + + +class MemcacheClientError(MemcacheError): + """Raised when memcached fails to parse the arguments to a request, likely + due to a malformed key and/or value, a bug in this library, or a version + mismatch with memcached.""" + pass + + +class MemcacheUnknownCommandError(MemcacheClientError): + """Raised when memcached fails to parse a request, likely due to a bug in + this library or a version mismatch with memcached.""" + pass + + +class MemcacheIllegalInputError(MemcacheClientError): + """Raised when a key or value is not legal for Memcache (see the class docs + for Client for more details).""" + pass + + +class MemcacheServerError(MemcacheError): + """Raised when memcached reports a failure while processing a request, + likely due to a bug or transient issue in memcached.""" + pass + + +class MemcacheUnknownError(MemcacheError): + """Raised when this library receives a response from memcached that it + cannot parse, likely due to a bug in this library or a version mismatch + with memcached.""" + pass + + +class MemcacheUnexpectedCloseError(MemcacheServerError): + "Raised when the connection with memcached closes unexpectedly." + pass diff --git a/pymemcache/test/conftest.py b/pymemcache/test/conftest.py index 369c2fd..9928a22 100644 --- a/pymemcache/test/conftest.py +++ b/pymemcache/test/conftest.py @@ -51,3 +51,17 @@ def pytest_generate_tests(metafunc): socket_modules.append(gevent_socket) metafunc.parametrize("socket_module", socket_modules) + + if 'client_class' in metafunc.fixturenames: + from pymemcache.client.base import PooledClient, Client + from pymemcache.client.hash import HashClient + + class HashClientSingle(HashClient): + def __init__(self, server, *args, **kwargs): + super(HashClientSingle, self).__init__( + [server], *args, **kwargs + ) + + metafunc.parametrize( + "client_class", [Client, PooledClient, HashClientSingle] + ) diff --git a/pymemcache/test/test_client.py b/pymemcache/test/test_client.py index 1efb030..29a71b1 100644 --- a/pymemcache/test/test_client.py +++ b/pymemcache/test/test_client.py @@ -19,10 +19,15 @@ import socket import unittest import pytest -from pymemcache.client import PooledClient -from pymemcache.client import Client, MemcacheUnknownCommandError -from pymemcache.client import MemcacheClientError, MemcacheServerError -from pymemcache.client import MemcacheUnknownError, MemcacheIllegalInputError +from pymemcache.client.base import PooledClient, Client +from pymemcache.exceptions import ( + MemcacheClientError, + MemcacheServerError, + MemcacheUnknownCommandError, + MemcacheUnknownError, + MemcacheIllegalInputError +) + from pymemcache import pool from pymemcache.test.utils import MockMemcacheClient diff --git a/pymemcache/test/test_client_hash.py b/pymemcache/test/test_client_hash.py new file mode 100644 index 0000000..0efa342 --- /dev/null +++ b/pymemcache/test/test_client_hash.py @@ -0,0 +1,111 @@ +from pymemcache.client.hash import HashClient +from pymemcache.client.base import Client, PooledClient +from pymemcache.exceptions import MemcacheUnknownError +from pymemcache import pool + +from .test_client import ClientTestMixin, MockSocket +import unittest +import pytest + + +class TestHashClient(ClientTestMixin, unittest.TestCase): + + def make_client_pool(self, hostname, mock_socket_values, serializer=None): + mock_client = Client(hostname, serializer=serializer) + mock_client.sock = MockSocket(mock_socket_values) + client = PooledClient(hostname, serializer=serializer) + client.client_pool = pool.ObjectPool(lambda: mock_client) + return mock_client + + def make_client(self, *mock_socket_values, **kwargs): + current_port = 11012 + client = HashClient([], **kwargs) + ip = '127.0.0.1' + + for vals in mock_socket_values: + s = '%s:%s' % (ip, current_port) + c = self.make_client_pool( + (ip, current_port), + vals + ) + client.clients[s] = c + client.hasher.add_node(s) + current_port += 1 + + return client + + def test_get_many_all_found(self): + client = self.make_client(*[ + [b'STORED\r\n', b'VALUE key3 0 6\r\nvalue2\r\nEND\r\n', ], + [b'STORED\r\n', b'VALUE key1 0 6\r\nvalue1\r\nEND\r\n', ], + ]) + + def get_clients(key): + if key == b'key3': + return client.clients['127.0.0.1:11012'] + else: + return client.clients['127.0.0.1:11013'] + + client._get_client = get_clients + + result = client.set(b'key1', b'value1', noreply=False) + result = client.set(b'key3', b'value2', noreply=False) + result = client.get_many([b'key1', b'key3']) + assert result == {b'key1': b'value1', b'key3': b'value2'} + + def test_get_many_some_found(self): + client = self.make_client(*[ + [b'END\r\n', ], + [b'STORED\r\n', b'VALUE key1 0 6\r\nvalue1\r\nEND\r\n', ], + ]) + + def get_clients(key): + if key == b'key3': + return client.clients['127.0.0.1:11012'] + else: + return client.clients['127.0.0.1:11013'] + + client._get_client = get_clients + result = client.set(b'key1', b'value1', noreply=False) + result = client.get_many([b'key1', b'key3']) + + assert result == {b'key1': b'value1'} + + def test_get_many_bad_server_data(self): + client = self.make_client(*[ + [b'STORED\r\n', b'VAXLUE key3 0 6\r\nvalue2\r\nEND\r\n', ], + [b'STORED\r\n', b'VAXLUE key1 0 6\r\nvalue1\r\nEND\r\n', ], + ]) + + def get_clients(key): + if key == b'key3': + return client.clients['127.0.0.1:11012'] + else: + return client.clients['127.0.0.1:11013'] + + client._get_client = get_clients + + with pytest.raises(MemcacheUnknownError): + client.set(b'key1', b'value1', noreply=False) + client.set(b'key3', b'value2', noreply=False) + client.get_many([b'key1', b'key3']) + + def test_get_many_bad_server_data_ignore(self): + client = self.make_client(*[ + [b'STORED\r\n', b'VAXLUE key3 0 6\r\nvalue2\r\nEND\r\n', ], + [b'STORED\r\n', b'VAXLUE key1 0 6\r\nvalue1\r\nEND\r\n', ], + ], ignore_exc=True) + + def get_clients(key): + if key == b'key3': + return client.clients['127.0.0.1:11012'] + else: + return client.clients['127.0.0.1:11013'] + + client._get_client = get_clients + + client.set(b'key1', b'value1', noreply=False) + client.set(b'key3', b'value2', noreply=False) + result = client.get_many([b'key1', b'key3']) + assert result == {} + # TODO: Test failover logic diff --git a/pymemcache/test/test_integration.py b/pymemcache/test/test_integration.py index 52576fe..491512a 100644 --- a/pymemcache/test/test_integration.py +++ b/pymemcache/test/test_integration.py @@ -16,13 +16,16 @@ import json import pytest import six -from pymemcache.client import (Client, MemcacheClientError) -from pymemcache.client import MemcacheIllegalInputError +from pymemcache.client.base import Client +from pymemcache.exceptions import ( + MemcacheIllegalInputError, + MemcacheClientError +) @pytest.mark.integration() -def test_get_set(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_get_set(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() result = client.get('key') @@ -44,8 +47,8 @@ def test_get_set(host, port, socket_module): @pytest.mark.integration() -def test_add_replace(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_add_replace(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() result = client.add(b'key', b'value', noreply=False) @@ -70,8 +73,8 @@ def test_add_replace(host, port, socket_module): @pytest.mark.integration() -def test_append_prepend(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_append_prepend(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() result = client.append(b'key', b'value', noreply=False) @@ -98,10 +101,9 @@ def test_append_prepend(host, port, socket_module): @pytest.mark.integration() -def test_cas(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_cas(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() - result = client.cas(b'key', b'value', b'1', noreply=False) assert result is None @@ -122,8 +124,8 @@ def test_cas(host, port, socket_module): @pytest.mark.integration() -def test_gets(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_gets(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() result = client.gets(b'key') @@ -136,8 +138,8 @@ def test_gets(host, port, socket_module): @pytest.mark.delete() -def test_delete(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_delete(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() result = client.delete(b'key', noreply=False) @@ -154,7 +156,7 @@ def test_delete(host, port, socket_module): @pytest.mark.integration() -def test_incr_decr(host, port, socket_module): +def test_incr_decr(client_class, host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() @@ -182,7 +184,7 @@ def test_incr_decr(host, port, socket_module): @pytest.mark.integration() -def test_misc(host, port, socket_module): +def test_misc(client_class, host, port, socket_module): client = Client((host, port), socket_module=socket_module) client.flush_all() @@ -208,8 +210,8 @@ def test_serialization_deserialization(host, port, socket_module): @pytest.mark.integration() -def test_errors(host, port, socket_module): - client = Client((host, port), socket_module=socket_module) +def test_errors(client_class, host, port, socket_module): + client = client_class((host, port), socket_module=socket_module) client.flush_all() def _key_with_ws(): diff --git a/pymemcache/test/test_rendezvous.py b/pymemcache/test/test_rendezvous.py new file mode 100644 index 0000000..067b760 --- /dev/null +++ b/pymemcache/test/test_rendezvous.py @@ -0,0 +1,203 @@ +from pymemcache.client.rendezvous import RendezvousHash +import pytest + + +@pytest.mark.unit() +def test_init_no_options(): + rendezvous = RendezvousHash() + assert 0 == len(rendezvous.nodes) + assert 1361238019 == rendezvous.hash_function('6666') + + +@pytest.mark.unit() +def test_init(): + nodes = ['0', '1', '2'] + rendezvous = RendezvousHash(nodes=nodes) + assert 3 == len(rendezvous.nodes) + assert 1361238019 == rendezvous.hash_function('6666') + + +@pytest.mark.unit() +def test_seed(): + rendezvous = RendezvousHash(seed=10) + assert 2981722772 == rendezvous.hash_function('6666') + + +@pytest.mark.unit() +def test_add_node(): + rendezvous = RendezvousHash() + rendezvous.add_node('1') + + assert 1 == len(rendezvous.nodes) + rendezvous.add_node('1') + + assert 1 == len(rendezvous.nodes) + rendezvous.add_node('2') + + assert 2 == len(rendezvous.nodes) + rendezvous.add_node('1') + + assert 2 == len(rendezvous.nodes) + + +@pytest.mark.unit() +def test_remove_node(): + nodes = ['0', '1', '2'] + rendezvous = RendezvousHash(nodes=nodes) + rendezvous.remove_node('2') + + assert 2 == len(rendezvous.nodes) + + with pytest.raises(ValueError): + rendezvous.remove_node('2') + + assert 2 == len(rendezvous.nodes) + + rendezvous.remove_node('1') + assert 1 == len(rendezvous.nodes) + + rendezvous.remove_node('0') + assert 0 == len(rendezvous.nodes) + + +@pytest.mark.unit() +def test_get_node(): + nodes = ['0', '1', '2'] + rendezvous = RendezvousHash(nodes=nodes) + assert '0' == rendezvous.get_node('ok') + assert '1' == rendezvous.get_node('mykey') + assert '2' == rendezvous.get_node('wat') + + +@pytest.mark.unit() +def test_get_node_after_removal(): + nodes = ['0', '1', '2'] + rendezvous = RendezvousHash(nodes=nodes) + rendezvous.remove_node('1') + + assert '0' == rendezvous.get_node('ok') + assert '0' == rendezvous.get_node('mykey') + assert '2' == rendezvous.get_node('wat') + + +@pytest.mark.unit() +def test_get_node_after_addition(): + nodes = ['0', '1', '2'] + rendezvous = RendezvousHash(nodes=nodes) + assert '0' == rendezvous.get_node('ok') + assert '1' == rendezvous.get_node('mykey') + assert '2' == rendezvous.get_node('wat') + assert '2' == rendezvous.get_node('lol') + rendezvous.add_node('3') + + assert '0' == rendezvous.get_node('ok') + assert '1' == rendezvous.get_node('mykey') + assert '2' == rendezvous.get_node('wat') + assert '3' == rendezvous.get_node('lol') + + +@pytest.mark.unit() +def test_grow(): + rendezvous = RendezvousHash() + + placements = {} + + for i in range(10): + rendezvous.add_node(str(i)) + placements[str(i)] = [] + + for i in range(1000): + node = rendezvous.get_node(str(i)) + placements[node].append(i) + + new_placements = {} + + for i in range(20): + rendezvous.add_node(str(i)) + new_placements[str(i)] = [] + + for i in range(1000): + node = rendezvous.get_node(str(i)) + new_placements[node].append(i) + + keys = [k for sublist in placements.values() for k in sublist] + new_keys = [k for sublist in new_placements.values() for k in sublist] + assert sorted(keys) == sorted(new_keys) + + added = 0 + removed = 0 + + for node, assignments in new_placements.items(): + after = set(assignments) + before = set(placements.get(node, [])) + removed += len(before.difference(after)) + added += len(after.difference(before)) + + assert added == removed + assert 1062 == (added + removed) + + +@pytest.mark.unit() +def test_shrink(): + rendezvous = RendezvousHash() + + placements = {} + for i in range(10): + rendezvous.add_node(str(i)) + placements[str(i)] = [] + + for i in range(1000): + node = rendezvous.get_node(str(i)) + placements[node].append(i) + + rendezvous.remove_node('9') + new_placements = {} + for i in range(9): + new_placements[str(i)] = [] + + for i in range(1000): + node = rendezvous.get_node(str(i)) + new_placements[node].append(i) + + keys = [k for sublist in placements.values() for k in sublist] + new_keys = [k for sublist in new_placements.values() for k in sublist] + assert sorted(keys) == sorted(new_keys) + + added = 0 + removed = 0 + for node, assignments in placements.items(): + after = set(assignments) + before = set(new_placements.get(node, [])) + removed += len(before.difference(after)) + added += len(after.difference(before)) + + assert added == removed + assert 202 == (added + removed) + + +def collide(key, seed): + return 1337 + + +@pytest.mark.unit() +def test_rendezvous_collision(): + nodes = ['c', 'b', 'a'] + rendezvous = RendezvousHash(nodes, hash_function=collide) + + for i in range(1000): + assert 'c' == rendezvous.get_node(i) + + +@pytest.mark.unit() +def test_rendezvous_names(): + nodes = [1, 2, 3, 'a', 'b', 'lol.wat.com'] + rendezvous = RendezvousHash(nodes, hash_function=collide) + + for i in range(10): + assert 'lol.wat.com' == rendezvous.get_node(i) + + nodes = [1, 'a', '0'] + rendezvous = RendezvousHash(nodes, hash_function=collide) + + for i in range(10): + assert 'a' == rendezvous.get_node(i) diff --git a/pymemcache/test/utils.py b/pymemcache/test/utils.py index 77b7c22..9661426 100644 --- a/pymemcache/test/utils.py +++ b/pymemcache/test/utils.py @@ -9,7 +9,7 @@ import time import six -from pymemcache.client import MemcacheIllegalInputError +from pymemcache.exceptions import MemcacheIllegalInputError class MockMemcacheClient(object): @@ -2,15 +2,16 @@ import os from setuptools import setup, find_packages - from pymemcache import __version__ + def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() readme = read('README.md') changelog = read('ChangeLog.rst') + setup( name='pymemcache', version=__version__, @@ -27,6 +28,9 @@ setup( 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: PyPy', + 'Programming Language :: Python :: PyPy3', 'License :: OSI Approved :: Apache Software License', 'Topic :: Database', ], @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, pypy, py33, py34, docs, flake8 +envlist = py26, py27, pypy, pypy3, py33, py34, docs, flake8 [testenv] commands = @@ -14,6 +14,6 @@ commands = [testenv:docs] commands = - pip install -r docs-requirements.txt + pip install -r docs-requirements.txt -r test-requirements.txt sphinx-apidoc -o docs/apidoc/ pymemcache sphinx-build -b html docs/ docs/_build |