diff options
| -rwxr-xr-x | .hgignore | 2 | ||||
| -rw-r--r-- | README.rst | 268 | ||||
| -rw-r--r-- | docs/build/Makefile | 95 | ||||
| -rw-r--r-- | docs/build/api.rst | 23 | ||||
| -rw-r--r-- | docs/build/builder.py | 9 | ||||
| -rw-r--r-- | docs/build/conf.py | 207 | ||||
| -rw-r--r-- | docs/build/index.rst | 28 | ||||
| -rw-r--r-- | docs/build/make.bat | 113 | ||||
| -rw-r--r-- | docs/build/usage.rst | 289 | ||||
| -rw-r--r-- | dogpile/cache/api.py | 22 | ||||
| -rw-r--r-- | dogpile/cache/region.py | 19 | ||||
| -rw-r--r-- | setup.py | 2 |
12 files changed, 804 insertions, 273 deletions
@@ -1,6 +1,6 @@ syntax:regexp ^build/ -^doc/build/output +^docs/build/output .pyc$ .orig$ .egg-info @@ -9,269 +9,5 @@ and use; users are encouraged to adapt the provided backends for their own needs, as high volume caching requires lots of tweaks and adjustments specific to an application and its environment. -Usage ------ - -A dogpile.cache configuration consists of the following components: - -* A *region*, which is an instance of ``CacheRegion``, and defines the configuration - details for a particular cache backend. -* A *backend*, which is an instance of ``CacheBackend``, describing how values - are stored and retrieved from a backend. This interface specifies only - ``get()``, ``put()`` and ``delete()``. -* Value generation functions. These are user-defined functions that generate - new values to be placed in the cache. - -The most common caching style in use these days is via memcached, so an example -of this using the `pylibmc <http://pypi.python.org/pypi/pylibmc>`_ backend looks like:: - - from dogpile.cache import make_region - - region = make_region().configure( - 'dogpile.cache.pylibmc', - expiration_time = 3600, - arguments = { - 'url':["127.0.0.1"], - 'binary':True, - 'behaviors':{"tcp_nodelay": True,"ketama":True} - } - ) - - @region.cache_on_arguments - def load_user_info(user_id): - return some_database.lookup_user_by_id(user_id) - -Above, we create a ``CacheRegion`` using the ``make_region()`` function, then -apply the backend configuration via the ``configure()`` method, which returns the -region. The name of the backend is the only required argument, -in this case ``dogpile.cache.pylibmc``. - -Subsequent arguments then include *expiration_time*, which is the expiration -time passed to the Dogpile lock, and *arguments*, which are arguments used directly -by the backend - in this case we are using arguments that are passed directly -to the pylibmc module. - -Backends --------- - -Backends are located using the setuptools entrypoint system. To make life easier -for writers of ad-hoc backends, a helper function is included which registers any -backend in the same way as if it were part of the existing sys.path. - -For example, to create a backend called ``DictionaryBackend``, we subclass -``CacheBackend``:: - - from dogpile.cache import CacheBackend, NO_VALUE - - class DictionaryBackend(CacheBackend): - def __init__(self, arguments): - self.cache = {} - - def get(self, key): - return self.cache.get(key, NO_VALUE) - - def put(self, key, value): - self.cache[key] = value - - def delete(self, key): - self.cache.pop(key) - -Then make sure the class is available underneath the entrypoint -``dogpile.cache``. If we did this in a ``setup.py`` file, it would be -in ``setup()`` as:: - - entry_points=""" - [dogpile.cache] - dictionary = mypackage.mybackend:DictionaryBackend - """ - -Alternatively, if we want to register the plugin in the same process -space without bothering to install anything, we can use ``register_backend``:: - - from dogpile.cache import register_backend - - register_backend("dictionary", "mypackage.mybackend", "DictionaryBackend") - -Our new backend would be usable in a region like this:: - - from dogpile.cache import make_region - - region = make_region("dictionary") - - data = region.put("somekey", "somevalue") - -The values we receive for the backend here are instances of -``CachedValue``. This is a tuple subclass of length two, of the form:: - - (payload, metadata) - -Where "payload" is the thing being cached, and "metadata" is information -we store in the cache - a dictionary which currently has just the "creation time" -and a "version identifier" as key/values. If the cache backend requires serialization, -pickle or similar can be used on the tuple - the "metadata" portion will always -be a small and easily serializable Python structure. - -Region Arguments ----------------- - -The ``make_region()`` function accepts these arguments: - -``name`` - - Optional. A string name for the region. This isn't used internally - but can be accessed via the ``.name`` parameter, helpful - for configuring a region from a config file. - -``function_key_generator`` - - Optional. Plug in a function that will produce a "cache key" given - a data creation function and arguments. The structure of this function - should be two levels: given the data creation function, return a new - function that generates the key based on the given arguments. Such - as:: - - def my_key_generator(fn): - namespace = fn.__name__ - def generate_key(*arg): - return namespace + "_".join(str(s) for s in arg) - return generate_key - - - region = make_region( - function_key_generator = my_key_generator - ).configure( - "dogpile.cache.dbm", - expiration_time=300, - arguments={ - "filename":"file.dbm" - } - ) - -``key_mangler`` - - Optional. Function which will "mangle" the incoming keys. If left - at ``None``, the backend may provide a default "mangler" function. - Set to ``False`` to unconditionally disable key mangling. - -One you have a ``CacheRegion``, the ``cache_on_arguments()`` method can -be used to decorate functions, but the cache itself can't be used until -``configure()`` is called. That method accepts these arguments: - -``backend`` - Required. This is the name of the ``CacheBackend`` to use, and - is resolved by loading the class from the ``dogpile.cache`` entrypoint. - -``expiration_time`` - - Optional. The expiration time passed to the dogpile system. The ``get_or_create()`` - method as well as the ``cache_on_arguments()`` decorator (note: **not** the - ``get()`` method) will call upon the value creation function after this - time period has passed since the last generation. - -``arguments`` - - Optional. The structure here is passed directly to the constructor - of the ``CacheBackend`` in use, though is typically a dictionary. - -Configure Region from a Configuration Dictionary ------------------------------------------------- - -Call ``configure_from_config()`` instead:: - - local_region = make_region() - memcached_region = make_region() - - # regions are ready to use for function - # decorators, but not yet for actual caching - - # later, when config is available - myconfig = { - "cache.local.backend":"dogpile.cache.dbm", - "cache.local.arguments.filename":"/path/to/dbmfile.dbm", - "cache.memcached.backend":"dogpile.cache.pylibmc", - "cache.memcached.arguments.url":"127.0.0.1, 10.0.0.1", - } - local_region.configure_from_config(myconfig, "cache.local.") - memcached_region.configure_from_config(myconfig, "cache.memcached.") - -Using a Region --------------- - -The ``CacheRegion`` object is our front-end interface to a cache. It includes -the following methods: - -``get(key)`` - - Return a value from the cache, based on the given key. While it's typical - the key is a string, it's passed through to the underlying backend so can - be of any type recognized by the backend. If the value is not present, returns the - token ``NO_VALUE``. ``NO_VALUE`` evaluates to False, but is separate - from ``None`` to distinguish between a cached value of ``None``. - Note that the ``expiration_time`` argument is **not** used here - this method - is a direct line to the backend's behavior. - -``get_or_create(key, creator)`` - - Similar to ``get``, will use the given "creation" function to create a new - value if the value does not exist. This will use the underlying dogpile/ - expiration mechanism to determine when/how the creation function is called. - -``put(key, value)`` - - Place a new value in the cache under the given key. - -``delete(key)`` - - Remove a value from the cache. This operation is idempotent (can be - called multiple times, or on a non-existent key, safely) - -``cache_on_arguments(fn)`` - - A function decorator that will cache the return value of the function - using a key derived from the name of the function, its location within - the application (i.e. source filename) as well as the arguments - passed to the function. - - The generation of the key from the function is the big - controversial thing that was a source of user issues with Beaker. Dogpile - provides the latest and greatest algorithm used by Beaker, but also - allows you to use whatever function you want, by specifying it - to ``make_region()`` using the ``function_key_generator`` argument. - - -Mako Integration ----------------- - -dogpile.cache includes a Mako plugin that replaces Beaker as the cache backend. -Simply setup a Mako template lookup using the "dogpile.cache" cache implementation -and a region dictionary:: - - from dogpile.cache import make_region - from mako.lookup import TemplateLookup - - my_regions = { - "local":make_region( - "dogpile.cache.dbm", - expiration_time=360, - arguments={"filename":"file.dbm"} - ) - "memcached":make_region( - "dogpile.cache.pylibmc", - expiration_time=3600, - arguments={"url":["127.0.0.1"]} - ) - } - - mako_lookup = TemplateLookup( - directories=["/myapp/templates"], - cache_impl="dogpile.cache", - cache_regions=my_regions - ) - -To use the above configuration in a template, use the ``cached=True`` argument on any -Mako tag which accepts it, in conjunction with the name of the desired region -as the ``cache_region`` argument:: - - <%def name="mysection()" cached=True cache_region="memcached"> - some content that's cached - </%def> +.. note:: dogpile.cache is **not released or completed** at this time. Development + is currently in progress and the current code is not yet functional. diff --git a/docs/build/Makefile b/docs/build/Makefile new file mode 100644 index 0000000..313a81e --- /dev/null +++ b/docs/build/Makefile @@ -0,0 +1,95 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = output + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dist-html same as html, but places files in /doc" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dist-html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) .. + @echo + @echo "Build finished. The HTML pages are in ../." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Alembic.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Alembic.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/build/api.rst b/docs/build/api.rst new file mode 100644 index 0000000..23c993f --- /dev/null +++ b/docs/build/api.rst @@ -0,0 +1,23 @@ +=== +API +=== + +Dogpile +======== + +.. automodule:: dogpile.dogpile + :members: + +NameRegistry +============= + +.. automodule:: dogpile.nameregistry + :members: + +Utilities +========== + +.. automodule:: dogpile.readwrite_lock + :members: + + diff --git a/docs/build/builder.py b/docs/build/builder.py new file mode 100644 index 0000000..37b97d6 --- /dev/null +++ b/docs/build/builder.py @@ -0,0 +1,9 @@ + +def autodoc_skip_member(app, what, name, obj, skip, options): + if what == 'class' and skip and name in ('__init__',) and obj.__doc__: + return False + else: + return skip + +def setup(app): + app.connect('autodoc-skip-member', autodoc_skip_member) diff --git a/docs/build/conf.py b/docs/build/conf.py new file mode 100644 index 0000000..234a854 --- /dev/null +++ b/docs/build/conf.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +# +# Dogpile.cache documentation build configuration file, created by +# sphinx-quickstart on Sat May 1 12:47:55 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# If your extensions are in another directory, add it here. If the directory +# is relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +sys.path.insert(0, os.path.abspath('../../')) + +import dogpile + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Dogpile' +copyright = u'2011, Mike Bayer' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = dogpile.__version__ +# The full version, including alpha/beta/rc tags. +release = dogpile.__version__ + + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'nature' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'dogpile.cachedoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'dogpile.cache.tex', u'Dogpile.Cache Documentation', + u'Mike Bayer', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +#{'python': ('http://docs.python.org/3.2', None)} + +intersphinx_mapping = {'sqla':('http://www.sqlalchemy.org/docs/', None)} diff --git a/docs/build/index.rst b/docs/build/index.rst new file mode 100644 index 0000000..ef9bcc0 --- /dev/null +++ b/docs/build/index.rst @@ -0,0 +1,28 @@ +========================================== +Welcome to Dogpile.Cache's documentation! +========================================== + +`dogpile.cache <http://bitbucket.org/zzzeek/dogpile.cache>`_ provides a simple +caching pattern based on the `dogpile <http://pypi.python.org/pypi/dogpile>`_ +locking system, including rudimentary backends. It effectively completes the +replacement of Beaker as far as caching is concerned, providing an open-ended +and simple pattern to configure caching. New backends are very easy to create +and use; users are encouraged to adapt the provided backends for their own +needs, as high volume caching requires lots of tweaks and adjustments specific +to an application and its environment. + + + +.. toctree:: + :maxdepth: 2 + + usage + api + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/build/make.bat b/docs/build/make.bat new file mode 100644 index 0000000..c334174 --- /dev/null +++ b/docs/build/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^<target^>` where ^<target^> is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Alembic.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Alembic.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/docs/build/usage.rst b/docs/build/usage.rst new file mode 100644 index 0000000..3fd8bcd --- /dev/null +++ b/docs/build/usage.rst @@ -0,0 +1,289 @@ +.. note:: dogpile.cache is **not released or completed** at this time. Development + is currently in progress and the current code is not yet functional. + +Introduction +============ + +At its core, Dogpile provides a locking interface around a "value creation" function. + +The interface supports several levels of usage, starting from +one that is very rudimentary, then providing more intricate +usage patterns to deal with certain scenarios. The documentation here will attempt to +provide examples that use successively more and more of these features, as +we approach how a fully featured caching system might be constructed around +Dogpile. + +Note that when using the `dogpile.cache <http://bitbucket.org/zzzeek/dogpile.cache>`_ +package, the constructs here provide the internal implementation for that system, +and users of that system don't need to access these APIs directly (though understanding +the general patterns is a terrific idea in any case). +Using the core Dogpile APIs described here directly implies you're building your own +resource-usage system outside, or in addition to, the one +`dogpile.cache <http://bitbucket.org/zzzeek/dogpile.cache>`_ provides. + +Usage +===== + +A dogpile.cache configuration consists of the following components: + +* A *region*, which is an instance of ``CacheRegion``, and defines the configuration + details for a particular cache backend. +* A *backend*, which is an instance of ``CacheBackend``, describing how values + are stored and retrieved from a backend. This interface specifies only + ``get()``, ``put()`` and ``delete()``. +* Value generation functions. These are user-defined functions that generate + new values to be placed in the cache. + +The most common caching style in use these days is via memcached, so an example +of this using the `pylibmc <http://pypi.python.org/pypi/pylibmc>`_ backend looks like:: + + from dogpile.cache import make_region + + region = make_region().configure( + 'dogpile.cache.pylibmc', + expiration_time = 3600, + arguments = { + 'url':["127.0.0.1"], + 'binary':True, + 'behaviors':{"tcp_nodelay": True,"ketama":True} + } + ) + + @region.cache_on_arguments + def load_user_info(user_id): + return some_database.lookup_user_by_id(user_id) + +Above, we create a ``CacheRegion`` using the ``make_region()`` function, then +apply the backend configuration via the ``configure()`` method, which returns the +region. The name of the backend is the only required argument, +in this case ``dogpile.cache.pylibmc``. + +Subsequent arguments then include *expiration_time*, which is the expiration +time passed to the Dogpile lock, and *arguments*, which are arguments used directly +by the backend - in this case we are using arguments that are passed directly +to the pylibmc module. + +Backends +======== + +Backends are located using the setuptools entrypoint system. To make life easier +for writers of ad-hoc backends, a helper function is included which registers any +backend in the same way as if it were part of the existing sys.path. + +For example, to create a backend called ``DictionaryBackend``, we subclass +``CacheBackend``:: + + from dogpile.cache import CacheBackend, NO_VALUE + + class DictionaryBackend(CacheBackend): + def __init__(self, arguments): + self.cache = {} + + def get(self, key): + return self.cache.get(key, NO_VALUE) + + def put(self, key, value): + self.cache[key] = value + + def delete(self, key): + self.cache.pop(key) + +Then make sure the class is available underneath the entrypoint +``dogpile.cache``. If we did this in a ``setup.py`` file, it would be +in ``setup()`` as:: + + entry_points=""" + [dogpile.cache] + dictionary = mypackage.mybackend:DictionaryBackend + """ + +Alternatively, if we want to register the plugin in the same process +space without bothering to install anything, we can use ``register_backend``:: + + from dogpile.cache import register_backend + + register_backend("dictionary", "mypackage.mybackend", "DictionaryBackend") + +Our new backend would be usable in a region like this:: + + from dogpile.cache import make_region + + region = make_region("dictionary") + + data = region.put("somekey", "somevalue") + +The values we receive for the backend here are instances of +``CachedValue``. This is a tuple subclass of length two, of the form:: + + (payload, metadata) + +Where "payload" is the thing being cached, and "metadata" is information +we store in the cache - a dictionary which currently has just the "creation time" +and a "version identifier" as key/values. If the cache backend requires serialization, +pickle or similar can be used on the tuple - the "metadata" portion will always +be a small and easily serializable Python structure. + +Region Arguments +================ + +The ``make_region()`` function accepts these arguments: + +``name`` + + Optional. A string name for the region. This isn't used internally + but can be accessed via the ``.name`` parameter, helpful + for configuring a region from a config file. + +``function_key_generator`` + + Optional. Plug in a function that will produce a "cache key" given + a data creation function and arguments. The structure of this function + should be two levels: given the data creation function, return a new + function that generates the key based on the given arguments. Such + as:: + + def my_key_generator(fn): + namespace = fn.__name__ + def generate_key(*arg): + return namespace + "_".join(str(s) for s in arg) + return generate_key + + + region = make_region( + function_key_generator = my_key_generator + ).configure( + "dogpile.cache.dbm", + expiration_time=300, + arguments={ + "filename":"file.dbm" + } + ) + +``key_mangler`` + + Optional. Function which will "mangle" the incoming keys. If left + at ``None``, the backend may provide a default "mangler" function. + Set to ``False`` to unconditionally disable key mangling. + +One you have a ``CacheRegion``, the ``cache_on_arguments()`` method can +be used to decorate functions, but the cache itself can't be used until +``configure()`` is called. That method accepts these arguments: + +``backend`` + Required. This is the name of the ``CacheBackend`` to use, and + is resolved by loading the class from the ``dogpile.cache`` entrypoint. + +``expiration_time`` + + Optional. The expiration time passed to the dogpile system. The ``get_or_create()`` + method as well as the ``cache_on_arguments()`` decorator (note: **not** the + ``get()`` method) will call upon the value creation function after this + time period has passed since the last generation. + +``arguments`` + + Optional. The structure here is passed directly to the constructor + of the ``CacheBackend`` in use, though is typically a dictionary. + +Configure Region from a Configuration Dictionary +================================================ + +Call ``configure_from_config()`` instead:: + + local_region = make_region() + memcached_region = make_region() + + # regions are ready to use for function + # decorators, but not yet for actual caching + + # later, when config is available + myconfig = { + "cache.local.backend":"dogpile.cache.dbm", + "cache.local.arguments.filename":"/path/to/dbmfile.dbm", + "cache.memcached.backend":"dogpile.cache.pylibmc", + "cache.memcached.arguments.url":"127.0.0.1, 10.0.0.1", + } + local_region.configure_from_config(myconfig, "cache.local.") + memcached_region.configure_from_config(myconfig, "cache.memcached.") + +Using a Region +============== + +The ``CacheRegion`` object is our front-end interface to a cache. It includes +the following methods: + +``get(key)`` + + Return a value from the cache, based on the given key. While it's typical + the key is a string, it's passed through to the underlying backend so can + be of any type recognized by the backend. If the value is not present, returns the + token ``NO_VALUE``. ``NO_VALUE`` evaluates to False, but is separate + from ``None`` to distinguish between a cached value of ``None``. + Note that the ``expiration_time`` argument is **not** used here - this method + is a direct line to the backend's behavior. + +``get_or_create(key, creator)`` + + Similar to ``get``, will use the given "creation" function to create a new + value if the value does not exist. This will use the underlying dogpile/ + expiration mechanism to determine when/how the creation function is called. + +``put(key, value)`` + + Place a new value in the cache under the given key. + +``delete(key)`` + + Remove a value from the cache. This operation is idempotent (can be + called multiple times, or on a non-existent key, safely) + +``cache_on_arguments(fn)`` + + A function decorator that will cache the return value of the function + using a key derived from the name of the function, its location within + the application (i.e. source filename) as well as the arguments + passed to the function. + + The generation of the key from the function is the big + controversial thing that was a source of user issues with Beaker. Dogpile + provides the latest and greatest algorithm used by Beaker, but also + allows you to use whatever function you want, by specifying it + to ``make_region()`` using the ``function_key_generator`` argument. + + +Mako Integration +================ + +dogpile.cache includes a Mako plugin that replaces Beaker as the cache backend. +Simply setup a Mako template lookup using the "dogpile.cache" cache implementation +and a region dictionary:: + + from dogpile.cache import make_region + from mako.lookup import TemplateLookup + + my_regions = { + "local":make_region( + "dogpile.cache.dbm", + expiration_time=360, + arguments={"filename":"file.dbm"} + ) + "memcached":make_region( + "dogpile.cache.pylibmc", + expiration_time=3600, + arguments={"url":["127.0.0.1"]} + ) + } + + mako_lookup = TemplateLookup( + directories=["/myapp/templates"], + cache_impl="dogpile.cache", + cache_regions=my_regions + ) + +To use the above configuration in a template, use the ``cached=True`` argument on any +Mako tag which accepts it, in conjunction with the name of the desired region +as the ``cache_region`` argument:: + + <%def name="mysection()" cached=True cache_region="memcached"> + some content that's cached + </%def> diff --git a/dogpile/cache/api.py b/dogpile/cache/api.py index 94275d2..d774ce8 100644 --- a/dogpile/cache/api.py +++ b/dogpile/cache/api.py @@ -67,6 +67,28 @@ class CacheBackend(object): ) ) + def get_mutex(self, key): + """Return an optional mutexing object for the given key. + + This object need only provide an ``acquire()`` + and ``release()`` method. + + May return ``None``, in which case the dogpile + lock will use a regular ``threading.Lock`` + object to mutex concurrent threads for + value creation. The default implementation + returns ``None``. + + Different backends may want to provide various + kinds of "mutex" objects, such as those which + link to lock files, distributed mutexes, + memcached semaphores, timers, etc. Whatever + kind of system is best suited for the scope + and behavior of the caching backend. + + """ + return None + def get(self, key): """Retrieve a value from the cache. diff --git a/dogpile/cache/region.py b/dogpile/cache/region.py index 66cd3c6..d0a9954 100644 --- a/dogpile/cache/region.py +++ b/dogpile/cache/region.py @@ -1,4 +1,6 @@ from dogpile import Dogpile, NeedRegenerationException +from dogpile.nameregistry import NameRegistry + from dogpile.cache.util import function_key_generator, PluginLoader, memoized_property from dogpile.cache.api import NO_VALUE, CachedValue import time @@ -20,7 +22,7 @@ class CacheRegion(object): def __init__(self, name=None, function_key_generator=function_key_generator, - key_mangler=None, + key_mangler=None ): """Construct a new :class:`.CacheRegion`. @@ -28,7 +30,7 @@ class CacheRegion(object): :param name: Optional, name for the region. :function_key_generator: Optional, key generator used by :meth:`.CacheRegion.cache_on_arguments`. - :key_mangler: Function which will be used on all incoming + :param key_mangler: Function which will be used on all incoming keys before passing to the backend. Defaults to ``None``, in which case the key mangling function recommended by the cache backend will be used. A typical mangler @@ -36,7 +38,8 @@ class CacheRegion(object): which coerces keys into a SHA1 hash, so that the string length is fixed. To disable all key mangling, set to ``False``. - + :param lock_generator: Function which, given a cache key, + returns a mutexing object. """ self.function_key_generator = function_key_generator if key_mangler: @@ -74,11 +77,17 @@ class CacheRegion(object): ) else: self.backend = backend_cls(arguments) - self.dogpile_registry = Dogpile.registry(expiration_time) + self.dogpile_registry = NameRegistry(self._create_dogpile) if self.key_mangler is None: self.key_mangler = backend.key_mangler return self + def _create_dogpile(self, identifier): + return Dogpile( + expiration_time, + lock=self.backend.get_mutex(identifier) + ) + def configure_from_config(self, config_dict, prefix): """Configure from a configuration dictionary and a prefix.""" @@ -139,7 +148,7 @@ class CacheRegion(object): self.backend.put(key, value) return value - dogpile = self.dogpile_registry.get(key) + dogpile = self.dogpile_registry.get(key, lock=self.backend.get_mutex(key)) with dogpile.acquire(gen_value, value_and_created_fn=get_value) as value: return value @@ -32,7 +32,7 @@ setup(name='dogpile.cache', entry_points=""" [mako.cache] dogpile = dogpile.cache.plugins.mako:MakoPlugin - """ + """, zip_safe=False, install_requires=['dogpile>=0.1.0'], test_suite='nose.collector', |