diff options
| -rw-r--r-- | Makefile | 15 | ||||
| -rw-r--r-- | README.rst | 77 | ||||
| -rw-r--r-- | docs/Makefile | 159 | ||||
| -rw-r--r-- | docs/api.rst | 37 | ||||
| -rw-r--r-- | docs/conf.py | 285 | ||||
| -rw-r--r-- | docs/index.rst | 10 | ||||
| -rw-r--r-- | msgpack/_packer.pyx | 42 | ||||
| -rw-r--r-- | msgpack/_unpacker.pyx | 78 | ||||
| -rw-r--r-- | msgpack/fallback.py | 92 |
9 files changed, 675 insertions, 120 deletions
@@ -1,12 +1,19 @@ .PHONY: test all python3 -all: +all: cython python setup.py build_ext -i -f - python setup.py build sdist -python3: +doc-serve: all + cd docs && make serve + +doc: + cd docs && make zip + +cython: + cython msgpack/*.pyx + +python3: cython python3 setup.py build_ext -i -f - python3 setup.py build sdist test: py.test test @@ -1,6 +1,6 @@ -=========================== -MessagePack Python Binding -=========================== +======================= +MessagePack for Python +======================= :author: INADA Naoki :version: 0.3.0 @@ -9,19 +9,42 @@ MessagePack Python Binding .. image:: https://secure.travis-ci.org/msgpack/msgpack-python.png :target: https://travis-ci.org/#!/msgpack/msgpack-python -WHAT IT IS ----------- +What's this +------------ `MessagePack <http://msgpack.org/>`_ is a fast, compact binary serialization format, suitable for similar data to JSON. This package provides CPython bindings for reading and writing MessagePack data. -NOTE for msgpack 0.2.x users +Install +--------- +You can use ``pip`` or ``easy_install`` to install msgpack:: + + $ easy_install msgpack-python + or + $ pip install msgpack-python + +PyPy +^^^^^ + +msgpack-python provides pure python implementation. +PyPy can use this. + +Windows +^^^^^^^ + +When you can't use binary distribution, you need to install Visual Studio +or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support +amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) + +Without extension, using pure python implementation on CPython runs slowly. + +Note for msgpack 0.2.x users ---------------------------- The msgpack 0.3 have some incompatible changes. -The default value of ``use_list`` keyword argument is ``True`` from 0.3.x. +The default value of ``use_list`` keyword argument is ``True`` from 0.3. You should pass the argument explicitly for backward compatibility. `Unpacker.unpack()` and some unpack methods now raises `OutOfData` @@ -29,10 +52,10 @@ instead of `StopIteration`. `StopIteration` is used for iterator protocol only. -HOW TO USE +How to use ----------- -one-shot pack & unpack +One-shot pack & unpack ^^^^^^^^^^^^^^^^^^^^^^ Use ``packb`` for packing and ``unpackb`` for unpacking. @@ -60,7 +83,7 @@ You should always pass the ``use_list`` keyword argument. See performance issues Read the docstring for other options. -streaming unpacking +Streaming unpacking ^^^^^^^^^^^^^^^^^^^ ``Unpacker`` is a "streaming unpacker". It unpacks multiple objects from one @@ -82,7 +105,7 @@ stream (or from bytes provided through its ``feed`` method). print unpacked -packing/unpacking of custom data type +Packing/unpacking of custom data type ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is also possible to pack/unpack custom data types. Here is an example for @@ -118,7 +141,7 @@ It is also possible to pack/unpack custom data types. Here is an example for key-value pairs. -advanced unpacking control +Advanced unpacking control ^^^^^^^^^^^^^^^^^^^^^^^^^^ As an alternative to iteration, ``Unpacker`` objects provide ``unpack``, @@ -150,27 +173,8 @@ callback function: unpacker.skip(bytestream.write) worker.send(bytestream.getvalue()) -INSTALL ---------- -You can use ``pip`` or ``easy_install`` to install msgpack:: - - $ easy_install msgpack-python - or - $ pip install msgpack-python - - -Windows -^^^^^^^ -msgpack provides some binary distribution for Windows. -You can install msgpack without compiler with them. - -When you can't use binary distribution, you need to install Visual Studio -or Windows SDK on Windows. (NOTE: Visual C++ Express 2010 doesn't support -amd64. Windows SDK is recommanded way to build amd64 msgpack without any fee.) - - -PERFORMANCE NOTE ------------------ +Note about performance +------------------------ GC ^^ @@ -179,8 +183,8 @@ CPython's GC starts when growing allocated object. This means unpacking may cause useless GC. You can use ``gc.disable()`` when unpacking large message. -use_list -^^^^^^^^^ +`use_list` option +^^^^^^^^^^^^^^^^^^ List is the default sequence type of Python. But tuple is lighter than list. You can use ``use_list=False`` while unpacking when performance is important. @@ -190,13 +194,12 @@ Python's dict can't use list as key and MessagePack allows array for key of mapp Another way to unpacking such object is using ``object_pairs_hook``. -TEST +Test ---- MessagePack uses `pytest` for testing. Run test with following command: $ py.test - .. vim: filetype=rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..0869604 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,159 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -E -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/msgpack.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/msgpack.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/msgpack" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/msgpack" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +serve: html + cd _build/html && python3.3 -m http.server + +zip: html + cd _build/html && zip -r ../../../msgpack-doc.zip . diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..50a84c4 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,37 @@ +API reference +============= + +.. module:: msgpack + +.. autofunction:: pack + +:func:`dump` is alias for :func:`pack` + +.. autofunction:: packb + +:func:`dumps` is alias for :func:`packb` + +.. autofunction:: unpack + +:func:`load` is alias for :func:`unpack` + +.. autofunction:: unpackb + +:func:`loads` is alias for :func:`unpackb` + +.. autoclass:: Packer + :members: + +.. autoclass:: Unpacker + :members: + +exceptions +----------- + +These exceptions are accessible via `msgpack` package. +(For example, `msgpack.OutOfData` is shortcut for `msgpack.exceptions.OutOfData`) + +.. automodule:: msgpack.exceptions + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..fba09b7 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +# +# msgpack documentation build configuration file, created by +# sphinx-quickstart on Sun Feb 24 14:20:50 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'msgpack' +copyright = u'2013, INADA Naoki' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# The full version, including alpha/beta/rc tags. +version = release = '0.3' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' +today_fmt = "%Y-%m-%d" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinxdoc' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'msgpackdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'msgpack.tex', u'msgpack Documentation', + u'Author', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'msgpack', u'msgpack Documentation', + [u'Author'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'msgpack', u'msgpack Documentation', + u'Author', 'msgpack', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + + +# -- Options for Epub output --------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = u'msgpack' +epub_author = u'Author' +epub_publisher = u'Author' +epub_copyright = u'2013, Author' + +# The language of the text. It defaults to the language option +# or en if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +#epub_exclude_files = [] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..72d4499 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,10 @@ +msgpack document +================== + +`MessagePack <http://msgpack.org>`_ is a efficient format for inter +language data exchange. + +.. toctree:: + :maxdepth: 1 + + api diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index a5bc570..562c92c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -39,9 +39,10 @@ cdef int DEFAULT_RECURSE_LIMIT=511 cdef class Packer(object): - """MessagePack Packer + """ + MessagePack Packer - usage: + usage:: packer = Packer() astream.write(packer.pack(a)) @@ -49,13 +50,18 @@ cdef class Packer(object): Packer's constructor has some keyword arguments: - * *defaut* - Convert user type to builtin type that Packer supports. - See also simplejson's document. - * *encoding* - Convert unicode to bytes with this encoding. (default: 'utf-8') - * *unicode_erros* - Error handler for encoding unicode. (default: 'strict') - * *use_single_float* - Use single precision float type for float. (default: False) - * *autoreset* - Reset buffer after each pack and return it's content as `bytes`. (default: True). - If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + :param callable default: + Convert user type to builtin type that Packer supports. + See also simplejson's document. + :param str encoding: + Convert unicode to bytes with this encoding. (default: 'utf-8') + :param str unicode_erros: + Error handler for encoding unicode. (default: 'strict') + :param bool use_single_float: + Use single precision float type for float. (default: False) + :param bool autoreset: + Reset buffer after each pack and return it's content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. """ cdef msgpack_packer pk cdef object _default @@ -75,6 +81,8 @@ cdef class Packer(object): self.pk.length = 0 def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, bint autoreset=1): + """ + """ self.use_float = use_single_float self.autoreset = autoreset if default is not None: @@ -218,7 +226,7 @@ cdef class Packer(object): Pack *pairs* as msgpack map type. *pairs* should sequence of pair. - (`len(pairs)` and `for k, v in *pairs*:` should be supported.) + (`len(pairs)` and `for k, v in pairs:` should be supported.) """ cdef int ret = msgpack_pack_map(&self.pk, len(pairs)) if ret == 0: @@ -245,15 +253,21 @@ cdef class Packer(object): return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) -def pack(object o, object stream, default=None, encoding='utf-8', unicode_errors='strict'): +def pack(object o, object stream, default=None, str encoding='utf-8', str unicode_errors='strict'): + """ + pack an object `o` and write it to stream) + + See :class:`Packer` for options. """ - pack an object `o` and write it to stream).""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors) stream.write(packer.pack(o)) -def packb(object o, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False): +def packb(object o, default=None, encoding='utf-8', str unicode_errors='strict', bint use_single_float=False): + """ + pack o and return packed bytes + + See :class:`Packer` for options. """ - pack o and return packed bytes.""" packer = Packer(default=default, encoding=encoding, unicode_errors=unicode_errors, use_single_float=use_single_float) return packer.pack(o) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index aeda02a..daeb6d7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -81,9 +81,12 @@ def unpackb(object packed, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, ): - """Unpack packed_bytes to object. Returns an unpacked object. + """ + Unpack packed_bytes to object. Returns an unpacked object. Raises `ValueError` when `packed` contains extra bytes. + + See :class:`Unpacker` for options. """ cdef template_context ctx cdef size_t off = 0 @@ -121,9 +124,12 @@ def unpack(object stream, object object_hook=None, object list_hook=None, bint use_list=1, encoding=None, unicode_errors="strict", object_pairs_hook=None, ): - """Unpack an object from `stream`. + """ + Unpack an object from `stream`. Raises `ValueError` when `stream` has extra bytes. + + See :class:`Unpacker` for options. """ return unpackb(stream.read(), use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, @@ -135,48 +141,58 @@ cdef class Unpacker(object): """ Streaming unpacker. - `file_like` is a file-like object having `.read(n)` method. - When `Unpacker` initialized with `file_like`, unpacker reads serialized data - from it and `.feed()` method is not usable. + arguments: - `read_size` is used as `file_like.read(read_size)`. - (default: min(1024**2, max_buffer_size)) + :param file_like: + File-like object having `.read(n)` method. + If specified, unpacker reads serialized data from it and :meth:`feed()` is not usable. - If `use_list` is true (default), msgpack list is deserialized to Python list. - Otherwise, it is deserialized to Python tuple. + :param int read_size: + Used as `file_like.read(read_size)`. (default: `min(1024**2, max_buffer_size)`) - `object_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it with a dict argument after deserializing a map. + :param bool use_list: + If true, unpack msgpack array to Python list. + Otherwise, unpack to Python tuple. (default: True) - `object_pairs_hook` is same to simplejson. If it is not None, it should be callable - and Unpacker calls it with a list of key-value pairs after deserializing a map. + :param callable object_hook: + When specified, it should be callable. + Unpacker calls it with a dict argument after unpacking msgpack map. + (See also simplejson) - `encoding` is encoding used for decoding msgpack bytes. If it is None (default), - msgpack bytes is deserialized to Python bytes. + :param callable object_pairs_hook: + When specified, it should be callable. + Unpacker calls it with a list of key-value pairs after unpacking msgpack map. + (See also simplejson) - `unicode_errors` is used for decoding bytes. + :param str encoding: + Encoding used for decoding msgpack raw. + If it is None (default), msgpack raw is deserialized to Python bytes. - `max_buffer_size` limits size of data waiting unpacked. - 0 means system's INT_MAX (default). - Raises `BufferFull` exception when it is insufficient. - You shoud set this parameter when unpacking data from untrasted source. + :param str unicode_errors: + Used for decoding msgpack raw with *encoding*. + (default: `'strict'`) + + :param int max_buffer_size: + Limits size of data waiting unpacked. 0 means system's INT_MAX (default). + Raises `BufferFull` exception when it is insufficient. + You shoud set this parameter when unpacking data from untrasted source. example of streaming deserialize from file-like object:: unpacker = Unpacker(file_like) for o in unpacker: - do_something(o) + process(o) example of streaming deserialize from socket:: unpacker = Unpacker() - while 1: + while True: buf = sock.recv(1024**2) if not buf: break unpacker.feed(buf) for o in unpacker: - do_something(o) + process(o) """ cdef template_context ctx cdef char* buf @@ -197,7 +213,7 @@ cdef class Unpacker(object): def __init__(self, file_like=None, Py_ssize_t read_size=0, bint use_list=1, object object_hook=None, object object_pairs_hook=None, object list_hook=None, - encoding=None, unicode_errors='strict', int max_buffer_size=0, + str encoding=None, str unicode_errors='strict', int max_buffer_size=0, ): cdef char *cenc=NULL, *cerr=NULL @@ -223,15 +239,17 @@ cdef class Unpacker(object): if encoding is not None: if isinstance(encoding, unicode): - encoding = encoding.encode('ascii') - self.encoding = encoding - cenc = PyBytes_AsString(encoding) + self.encoding = encoding.encode('ascii') + else: + self.encoding = encoding + cenc = PyBytes_AsString(self.encoding) if unicode_errors is not None: if isinstance(unicode_errors, unicode): - unicode_errors = unicode_errors.encode('ascii') - self.unicode_errors = unicode_errors - cerr = PyBytes_AsString(unicode_errors) + self.unicode_errors = unicode_errors.encode('ascii') + else: + self.unicode_errors = unicode_errors + cerr = PyBytes_AsString(self.unicode_errors) init_ctx(&self.ctx, object_hook, object_pairs_hook, list_hook, use_list, cenc, cerr) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index e295b18..f3247fc 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -61,46 +61,44 @@ TYPE_RAW = 3 DEFAULT_RECURSE_LIMIT=511 -def pack(o, stream, default=None, encoding='utf-8', unicode_errors='strict'): - """ Pack object `o` and write it to `stream` """ - packer = Packer(default=default, encoding=encoding, - unicode_errors=unicode_errors) +def pack(o, stream, **kwargs): + """ + Pack object `o` and write it to `stream` + + See :class:`Packer` for options. + """ + packer = Packer(**kwargs) stream.write(packer.pack(o)) -def packb(o, default=None, encoding='utf-8', unicode_errors='struct', - use_single_float=False): - """ Pack object `o` and return packed bytes """ - packer = Packer(default=default, - encoding=encoding, - unicode_errors=unicode_errors, - use_single_float=use_single_float) - return packer.pack(o) - -def unpack(stream, object_hook=None, list_hook=None, use_list=True, - encoding=None, unicode_errors='strict', - object_pairs_hook=None): - """ Unpack an object from `stream`. - - Raises `ExtraData` when `stream` has extra bytes. """ - unpacker = Unpacker(stream, object_hook=object_hook, list_hook=list_hook, - use_list=use_list, - encoding=encoding, unicode_errors=unicode_errors, - object_pairs_hook=object_pairs_hook) - ret = unpacker._unpack() - if unpacker._got_extradata(): - raise ExtraData(ret, unpacker._get_extradata()) +def packb(o, **kwargs): + """ + Pack object `o` and return packed bytes + + See :class:`Packer` for options. + """ + return Packer(**kwargs).pack(o) + +def unpack(stream, **kwargs): + """ + Unpack an object from `stream`. + + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(stream, **kwargs) + ret = unpacker._fb_unpack() + if unpacker._fb_got_extradata(): + raise ExtraData(ret, unpacker._fb_get_extradata()) return ret -def unpackb(packed, object_hook=None, list_hook=None, use_list=True, - encoding=None, unicode_errors='strict', - object_pairs_hook=None): - """ Unpack an object from `packed`. +def unpackb(packed, **kwargs): + """ + Unpack an object from `packed`. - Raises `ExtraData` when `packed` contains extra bytes. """ - unpacker = Unpacker(None, object_hook=object_hook, list_hook=list_hook, - use_list=use_list, - encoding=encoding, unicode_errors=unicode_errors, - object_pairs_hook=object_pairs_hook) + Raises `ExtraData` when `packed` contains extra bytes. + See :class:`Unpacker` for options. + """ + unpacker = Unpacker(None, **kwargs) unpacker.feed(packed) try: ret = unpacker._unpack() @@ -447,6 +445,30 @@ class Unpacker(object): class Packer(object): + """ + MessagePack Packer + + usage: + + packer = Packer() + astream.write(packer.pack(a)) + astream.write(packer.pack(b)) + + Packer's constructor has some keyword arguments: + + :param callable default: + Convert user type to builtin type that Packer supports. + See also simplejson's document. + :param str encoding: + Convert unicode to bytes with this encoding. (default: 'utf-8') + :param str unicode_erros: + Error handler for encoding unicode. (default: 'strict') + :param bool use_single_float: + Use single precision float type for float. (default: False) + :param bool autoreset: + Reset buffer after each pack and return it's content as `bytes`. (default: True). + If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. + """ def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=True): self._use_float = use_single_float |