diff options
| -rw-r--r-- | .gitignore | 5 | ||||
| -rw-r--r-- | docs/build/api.rst | 12 | ||||
| -rw-r--r-- | docs/build/changelog.rst | 28 | ||||
| -rw-r--r-- | dogpile/cache/backends/__init__.py | 1 | ||||
| -rw-r--r-- | dogpile/cache/backends/memory.py | 72 | ||||
| -rw-r--r-- | dogpile/cache/region.py | 36 | ||||
| -rw-r--r-- | dogpile/cache/util.py | 7 | ||||
| -rw-r--r-- | tests/cache/test_decorator.py | 18 | ||||
| -rw-r--r-- | tests/cache/test_memory_backend.py | 3 |
9 files changed, 153 insertions, 29 deletions
@@ -1,10 +1,11 @@ *.pyc -build/ +/build dist/ docs/build/output/ *.orig alembic.ini tox.ini .venv -.egg-info +*.egg +*.egg-info .coverage diff --git a/docs/build/api.rst b/docs/build/api.rst index d45493e..cfb9ffc 100644 --- a/docs/build/api.rst +++ b/docs/build/api.rst @@ -14,12 +14,12 @@ Backend API ============= See the section :ref:`creating_backends` for details on how to -register new backends or :ref:`changing_backend_behavior` for details on -how to alter the behavior of existing backends. +register new backends or :ref:`changing_backend_behavior` for details on +how to alter the behavior of existing backends. .. automodule:: dogpile.cache.api :members: - + Backends ========== @@ -35,11 +35,11 @@ Backends .. automodule:: dogpile.cache.backends.file :members: - + .. automodule:: dogpile.cache.proxy :members: - - + + Plugins ======== diff --git a/docs/build/changelog.rst b/docs/build/changelog.rst index 65984f5..2921f47 100644 --- a/docs/build/changelog.rst +++ b/docs/build/changelog.rst @@ -5,6 +5,34 @@ Changelog :version: 0.5.3 .. change:: + :tags: bug, py3k + + Fixed bug where the :meth:`.Region.get_multi` method wasn't calling + the backend correctly in Py3K (e.g. was passing a destructive ``map()`` + object) which would cause this method to fail on the memcached backend. + + .. change:: + :tags: feature + :tickets: 55 + + Added a ``get()`` method to complement the ``set()``, ``invalidate()`` + and ``refresh()`` methods established on functions decorated by + :meth:`.CacheRegion.cache_on_arguments` and + :meth:`.CacheRegion.cache_multi_on_arguments`. Pullreq courtesy + Eric Hanchrow. + + .. change:: + :tags: feature + :tickets: 51 + :pullreq: 11 + + Added a new variant on :class:`.MemoryBackend`, :class:`.MemoryPickleBackend`. + This backend applies ``pickle.dumps()`` and ``pickle.loads()`` to cached + values upon set and get, so that similar copy-on-cache behavior as that + of other backends is employed, guarding cached values against subsequent + in-memory state changes. Pullreq courtesy Jonathan Vanasco. + + .. change:: :tags: bug :pullreq: 9 diff --git a/dogpile/cache/backends/__init__.py b/dogpile/cache/backends/__init__.py index a0c111b..a9b9c2e 100644 --- a/dogpile/cache/backends/__init__.py +++ b/dogpile/cache/backends/__init__.py @@ -5,4 +5,5 @@ register_backend("dogpile.cache.pylibmc", "dogpile.cache.backends.memcached", "P register_backend("dogpile.cache.bmemcached", "dogpile.cache.backends.memcached", "BMemcachedBackend") register_backend("dogpile.cache.memcached", "dogpile.cache.backends.memcached", "MemcachedBackend") register_backend("dogpile.cache.memory", "dogpile.cache.backends.memory", "MemoryBackend") +register_backend("dogpile.cache.memory_pickle", "dogpile.cache.backends.memory", "MemoryPickleBackend") register_backend("dogpile.cache.redis", "dogpile.cache.backends.redis", "RedisBackend") diff --git a/dogpile/cache/backends/memory.py b/dogpile/cache/backends/memory.py index e606bea..cee989b 100644 --- a/dogpile/cache/backends/memory.py +++ b/dogpile/cache/backends/memory.py @@ -1,12 +1,17 @@ """ -Memory Backend --------------- +Memory Backends +--------------- -Provides a simple dictionary-based backend. +Provides simple dictionary-based backends. + +The two backends are :class:`.MemoryBackend` and :class:`.MemoryPickleBackend`; +the latter applies a serialization step to cached values while the former +places the value as given into the dictionary. """ from dogpile.cache.api import CacheBackend, NO_VALUE +from dogpile.cache.compat import pickle class MemoryBackend(CacheBackend): """A backend that uses a plain dictionary. @@ -41,23 +46,38 @@ class MemoryBackend(CacheBackend): """ + pickle_values = False + def __init__(self, arguments): self._cache = arguments.pop("cache_dict", {}) def get(self, key): - return self._cache.get(key, NO_VALUE) + value = self._cache.get(key, NO_VALUE) + if value is not NO_VALUE and self.pickle_values: + value = pickle.loads(value) + return value def get_multi(self, keys): - return [ - self._cache.get(key, NO_VALUE) - for key in keys - ] + ret = [self._cache.get(key, NO_VALUE) + for key in keys] + if self.pickle_values: + ret = [ + pickle.loads(value) + if value is not NO_VALUE else value + for value in ret + ] + return ret def set(self, key, value): + if self.pickle_values: + value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) self._cache[key] = value def set_multi(self, mapping): - for key,value in mapping.items(): + pickle_values = self.pickle_values + for key, value in mapping.items(): + if pickle_values: + value = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) self._cache[key] = value def delete(self, key): @@ -66,3 +86,37 @@ class MemoryBackend(CacheBackend): def delete_multi(self, keys): for key in keys: self._cache.pop(key, None) + + +class MemoryPickleBackend(MemoryBackend): + """A backend that uses a plain dictionary, but serializes objects on + :meth:`.MemoryBackend.set` and deserializes :meth:`.MemoryBackend.get`. + + E.g.:: + + from dogpile.cache import make_region + + region = make_region().configure( + 'dogpile.cache.memory_pickle' + ) + + The usage of pickle to serialize cached values allows an object + as placed in the cache to be a copy of the original given object, so + that any subsequent changes to the given object aren't reflected + in the cached value, thus making the backend behave the same way + as other backends which make use of serialization. + + The serialization is performed via pickle, and incurs the same + performance hit in doing so as that of other backends; in this way + the :class:`.MemoryPickleBackend` performance is somewhere in between + that of the pure :class:`.MemoryBackend` and the remote server oriented + backends such as that of Memcached or Redis. + + Pickle behavior here is the same as that of the Redis backend, using + either ``cPickle`` or ``pickle`` and specifying ``HIGHEST_PROTOCOL`` + upon serialize. + + .. versionadded:: 0.5.3 + + """ + pickle_values = True diff --git a/dogpile/cache/region.py b/dogpile/cache/region.py index bbe23e0..e7d152b 100644 --- a/dogpile/cache/region.py +++ b/dogpile/cache/region.py @@ -504,7 +504,7 @@ class CacheRegion(object): """ if self.key_mangler: - keys = map(lambda key: self.key_mangler(key), keys) + keys = list(map(lambda key: self.key_mangler(key), keys)) backend_values = self.backend.get_multi(keys) @@ -814,7 +814,7 @@ class CacheRegion(object): """ if self.key_mangler: - keys = map(lambda key: self.key_mangler(key), keys) + keys = list(map(lambda key: self.key_mangler(key), keys)) self.backend.delete_multi(keys) @@ -876,6 +876,15 @@ class CacheRegion(object): .. versionadded:: 0.5.0 Added ``refresh()`` method to decorated function. + Lastly, the ``get()`` method returns either the value cached + for the given key, or the token ``NO_VALUE`` if no such key + exists:: + + value = generate_something.get(5, 6) + + .. versionadded:: 0.5.3 Added ``get()`` method to decorated + function. + The default key generation will use the name of the function, the module name for the function, the arguments passed, as well as an optional "namespace" @@ -1006,6 +1015,10 @@ class CacheRegion(object): key = key_generator(*arg, **kw) self.set(key, value) + def get(*arg, **kw): + key = key_generator(*arg, **kw) + return self.get(key) + def refresh(*arg, **kw): key = key_generator(*arg, **kw) value = fn(*arg, **kw) @@ -1015,6 +1028,7 @@ class CacheRegion(object): decorate.set = set_ decorate.invalidate = invalidate decorate.refresh = refresh + decorate.get = get return decorate return decorator @@ -1070,17 +1084,25 @@ class CacheRegion(object): generate_something.set({"k1": "value1", "k2": "value2", "k3": "value3"}) - an ``invalidate()`` method, which has the effect of deleting + ...an ``invalidate()`` method, which has the effect of deleting the given sequence of keys using the same mechanism as that of :meth:`.CacheRegion.delete_multi`:: generate_something.invalidate("k1", "k2", "k3") - and finally a ``refresh()`` method, which will call the creation + ...a ``refresh()`` method, which will call the creation function, cache the new values, and return them:: values = generate_something.refresh("k1", "k2", "k3") + ...and a ``get()`` method, which will return values + based on the given arguments:: + + values = generate_something.get("k1", "k2", "k3") + + .. versionadded:: 0.5.3 Added ``get()`` method to decorated + function. + Parameters passed to :meth:`.CacheRegion.cache_multi_on_arguments` have the same meaning as those passed to :meth:`.CacheRegion.cache_on_arguments`. @@ -1174,6 +1196,10 @@ class CacheRegion(object): in zip(gen_keys, keys)) ) + def get(*arg): + keys = key_generator(*arg) + return self.get_multi(keys) + def refresh(*arg): keys = key_generator(*arg) values = fn(*arg) @@ -1191,6 +1217,7 @@ class CacheRegion(object): decorate.set = set_ decorate.invalidate = invalidate decorate.refresh = refresh + decorate.get = get return decorate return decorator @@ -1206,4 +1233,3 @@ def make_region(*arg, **kw): """ return CacheRegion(*arg, **kw) - diff --git a/dogpile/cache/util.py b/dogpile/cache/util.py index e61bbba..8f6f3ff 100644 --- a/dogpile/cache/util.py +++ b/dogpile/cache/util.py @@ -1,6 +1,5 @@ from hashlib import sha1 import inspect -import sys import re import collections from . import compat @@ -31,10 +30,8 @@ class PluginLoader(object): def load(self, name): if name in self.impls: - return self.impls[name]() - else: #pragma NO COVERAGE - # TODO: if someone has ideas on how to - # unit test entrypoint stuff, let me know. + return self.impls[name]() + else: # pragma NO COVERAGE import pkg_resources for impl in pkg_resources.iter_entry_points( self.group, diff --git a/tests/cache/test_decorator.py b/tests/cache/test_decorator.py index d2c142c..c16bdc3 100644 --- a/tests/cache/test_decorator.py +++ b/tests/cache/test_decorator.py @@ -99,6 +99,22 @@ class DecoratorTest(_GenericBackendFixture, TestCase): go.set(0, 1, 3) eq_(go(1, 3), 0) + def test_explicit_get(self): + go = self._fixture(expiration_time=1) + eq_(go(1, 2), (1, 1, 2)) + eq_(go.get(1, 2), (1, 1, 2)) + eq_(go.get(2, 1), NO_VALUE) + eq_(go(2, 1), (2, 2, 1)) + eq_(go.get(2, 1), (2, 2, 1)) + + def test_explicit_get_multi(self): + go = self._multi_fixture(expiration_time=1) + eq_(go(1, 2), ['1 1', '1 2']) + eq_(go.get(1, 2), ['1 1', '1 2']) + eq_(go.get(3, 1), [NO_VALUE, '1 1']) + eq_(go(3, 1), ['2 3', '1 1']) + eq_(go.get(3, 1), ['2 3', '1 1']) + def test_explicit_set_multi(self): go = self._multi_fixture(expiration_time=1) eq_(go(1, 2), ['1 1', '1 2']) @@ -385,5 +401,3 @@ class CacheDecoratorTest(_GenericBackendFixture, TestCase): generate.set({7: 18, 10: 15}) eq_(generate(2, 7, 10), ['2 5', 18, 15]) - - diff --git a/tests/cache/test_memory_backend.py b/tests/cache/test_memory_backend.py index 1746c7a..00c23b8 100644 --- a/tests/cache/test_memory_backend.py +++ b/tests/cache/test_memory_backend.py @@ -3,3 +3,6 @@ from ._fixtures import _GenericBackendTest class MemoryBackendTest(_GenericBackendTest): backend = "dogpile.cache.memory" + +class MemoryPickleBackendTest(_GenericBackendTest): + backend = "dogpile.cache.memory_pickle" |