diff options
47 files changed, 1330 insertions, 6135 deletions
@@ -18,3 +18,8 @@ profile # vi noise *.swp docs/_build/* +coverage.xml +nosetests.xml +junit-py25.xml +junit-py26.xml +junit-py27.xml
\ No newline at end of file @@ -12,4 +12,5 @@ Patches and Suggestions - Luke Lee - Josh Ourisman -- Luca Beltrame
\ No newline at end of file +- Luca Beltrame +- Benjamin Wohlwend
\ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index 95b9328..be61bb1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,16 @@ History ------- +0.9.4 (2011-02-18) +++++++++++++++++++ + +* Python 2.5 Support! +* Tox Testing for 2.5, 2.6, 2.7 +* AnyJSON Integrated +* OrderedDict support ? +* Caved to community pressure (spaces) + + 0.9.3 (2011-01-31) ++++++++++++++++++ @@ -60,29 +60,38 @@ SOFTWARE. -SimpleJSON License +AnyJSON License ================== -Copyright (c) 2006 Bob Ippolito +This software is licensed under the ``New BSD License``: -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: +Copyright (c) 2009, by the authors +All rights reserved. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +Neither the name of the authors nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/krstyle.sty b/docs/krstyle.sty new file mode 100644 index 0000000..b2d073d --- /dev/null +++ b/docs/krstyle.sty @@ -0,0 +1,118 @@ +\definecolor{TitleColor}{rgb}{0,0,0} +\definecolor{InnerLinkColor}{rgb}{0,0,0} + +\renewcommand{\maketitle}{% + \begin{titlepage}% + \let\footnotesize\small + \let\footnoterule\relax + \ifsphinxpdfoutput + \begingroup + % This \def is required to deal with multi-line authors; it + % changes \\ to ', ' (comma-space), making it pass muster for + % generating document info in the PDF file. + \def\\{, } + \pdfinfo{ + /Author (\@author) + /Title (\@title) + } + \endgroup + \fi + \begin{flushright}% + %\sphinxlogo% + {\center + \vspace*{3cm} + \includegraphics{logo.pdf} + \vspace{3cm} + \par + {\rm\Huge \@title \par}% + {\em\LARGE \py@release\releaseinfo \par} + {\large + \@date \par + \py@authoraddress \par + }}% + \end{flushright}%\par + \@thanks + \end{titlepage}% + \cleardoublepage% + \setcounter{footnote}{0}% + \let\thanks\relax\let\maketitle\relax + %\gdef\@thanks{}\gdef\@author{}\gdef\@title{} +} + +\fancypagestyle{normal}{ + \fancyhf{} + \fancyfoot[LE,RO]{{\thepage}} + \fancyfoot[LO]{{\nouppercase{\rightmark}}} + \fancyfoot[RE]{{\nouppercase{\leftmark}}} + \fancyhead[LE,RO]{{ \@title, \py@release}} + \renewcommand{\headrulewidth}{0.4pt} + \renewcommand{\footrulewidth}{0.4pt} +} + +\fancypagestyle{plain}{ + \fancyhf{} + \fancyfoot[LE,RO]{{\thepage}} + \renewcommand{\headrulewidth}{0pt} + \renewcommand{\footrulewidth}{0.4pt} +} + +\titleformat{\section}{\Large}% + {\py@TitleColor\thesection}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\subsection}{\large}% + {\py@TitleColor\thesubsection}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\subsubsection}{}% + {\py@TitleColor\thesubsubsection}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\paragraph}{\large}% + {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor} + +\ChNameVar{\raggedleft\normalsize} +\ChNumVar{\raggedleft \bfseries\Large} +\ChTitleVar{\raggedleft \rm\Huge} + +\renewcommand\thepart{\@Roman\c@part} +\renewcommand\part{% + \pagestyle{empty} + \if@noskipsec \leavevmode \fi + \cleardoublepage + \vspace*{6cm}% + \@afterindentfalse + \secdef\@part\@spart} + +\def\@part[#1]#2{% + \ifnum \c@secnumdepth >\m@ne + \refstepcounter{part}% + \addcontentsline{toc}{part}{\thepart\hspace{1em}#1}% + \else + \addcontentsline{toc}{part}{#1}% + \fi + {\parindent \z@ %\center + \interlinepenalty \@M + \normalfont + \ifnum \c@secnumdepth >\m@ne + \rm\Large \partname~\thepart + \par\nobreak + \fi + \MakeUppercase{\rm\Huge #2}% + \markboth{}{}\par}% + \nobreak + \vskip 8ex + \@afterheading} +\def\@spart#1{% + {\parindent \z@ %\center + \interlinepenalty \@M + \normalfont + \huge \bfseries #1\par}% + \nobreak + \vskip 3ex + \@afterheading} + +% use inconsolata font +\usepackage{inconsolata} + +% fix single quotes, for inconsolata. (does not work) +%%\usepackage{textcomp} +%%\begingroup +%% \catcode`'=\active +%% \g@addto@macro\@noligs{\let'\textsinglequote} +%% \endgroup +%%\endinput diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 72b8792..4349952 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -268,7 +268,7 @@ It's that simple. The original :class:`Dataset` is untouched. Excel Workbook With Multiple Sheets ------------------------------------ -When dealing with a large number of :class:`Datasets <Dataset>` in spreadsheet format, it's quite common to group multiple spreadsheets into a single Excel file, known as a Workbook. Tablib makes it extremely easy to build webooks with the handy, :class:`Databook` class. +When dealing with a large number of :class:`Datasets <Dataset>` in spreadsheet format, it's quite common to group multiple spreadsheets into a single Excel file, known as a Workbook. Tablib makes it extremely easy to build workbooks with the handy, :class:`Databook` class. Let's say we have 3 different :class:`Datasets <Dataset>`. All we have to do is add then to a :class:`Databook` object... :: @@ -3,15 +3,15 @@ from fabric.api import * def scrub(): - """ Death to the bytecode! """ - local('rm -fr dist build') - local("find . -name \"*.pyc\" -exec rm '{}' ';'") + """ Death to the bytecode! """ + local('rm -fr dist build') + local("find . -name \"*.pyc\" -exec rm '{}' ';'") def docs(): - """Build docs.""" - os.system('make dirhtml') - os.chdir('_build/dirhtml') - os.system('sphinxtogithub .') - os.system('git add -A') - os.system('git commit -m \'documentation update\'') - os.system('git push origin gh-pages')
\ No newline at end of file + """Build docs.""" + os.system('make dirhtml') + os.chdir('_build/dirhtml') + os.system('sphinxtogithub .') + os.system('git add -A') + os.system('git commit -m \'documentation update\'') + os.system('git push origin gh-pages')
\ No newline at end of file @@ -8,43 +8,45 @@ from distutils.core import setup def publish(): - """Publish to PyPi""" - os.system("python setup.py sdist upload") + """Publish to PyPi""" + os.system("python setup.py sdist upload") if sys.argv[-1] == "publish": - publish() - sys.exit() + publish() + sys.exit() required = [] +if sys.version_info < (2,6): + required.append('simplejson') + setup( - name='tablib', - version='0.9.3', - description='Format agnostic tabular data library (XLS, JSON, YAML, CSV)', - long_description=open('README.rst').read() + '\n\n' + - open('HISTORY.rst').read(), - author='Kenneth Reitz', - author_email='me@kennethreitz.com', - url='http://tablib.org', - packages= [ - 'tablib', 'tablib.formats', - 'tablib.packages', - 'tablib.packages.simplejson', - 'tablib.packages.xlwt', - 'tablib.packages.yaml', - ], - install_requires=required, - license='MIT', - classifiers=( - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Natural Language :: English', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - # 'Programming Language :: Python :: 2.5', + name='tablib', + version='0.9.4', + description='Format agnostic tabular data library (XLS, JSON, YAML, CSV)', + long_description=open('README.rst').read() + '\n\n' + + open('HISTORY.rst').read(), + author='Kenneth Reitz', + author_email='me@kennethreitz.com', + url='http://tablib.org', + packages= [ + 'tablib', 'tablib.formats', + 'tablib.packages', + 'tablib.packages.xlwt', + 'tablib.packages.yaml', + ], + install_requires=required, + license='MIT', + classifiers=( + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - # 'Programming Language :: Python :: 3.0', - # 'Programming Language :: Python :: 3.1', - ), + 'Programming Language :: Python :: 2.7', + # 'Programming Language :: Python :: 3.0', + # 'Programming Language :: Python :: 3.1', + ), ) diff --git a/tablib/__init__.py b/tablib/__init__.py index 3f23850..c2205b6 100644 --- a/tablib/__init__.py +++ b/tablib/__init__.py @@ -2,7 +2,7 @@ """ from tablib.core import ( - Databook, Dataset, detect, import_set, - InvalidDatasetType, InvalidDimensions, UnsupportedFormat + Databook, Dataset, detect, import_set, + InvalidDatasetType, InvalidDimensions, UnsupportedFormat ) diff --git a/tablib/core.py b/tablib/core.py index 5b224a0..48ffddb 100644 --- a/tablib/core.py +++ b/tablib/core.py @@ -14,83 +14,90 @@ from operator import itemgetter from tablib import formats +try: + from collections import OrderedDict +except ImportError: + from tablib.packages.ordereddict import OrderedDict + __title__ = 'tablib' -__version__ = '0.9.3' -__build__ = 0x000903 +__version__ = '0.9.4' +__build__ = 0x000904 __author__ = 'Kenneth Reitz' __license__ = 'MIT' __copyright__ = 'Copyright 2011 Kenneth Reitz' class Row(object): - """Internal Row object. Mainly used for filtering.""" + """Internal Row object. Mainly used for filtering.""" + + __slots__ = ['tuple', '_row', 'tags'] - __slots__ = ['tuple', '_row', 'tags'] + def __init__(self, row=list(), tags=list()): + self._row = list(row) + self.tags = list(tags) - def __init__(self, row=list(), tags=list()): - self._row = list(row) - self.tags = list(tags) + def __iter__(self): + return (col for col in self._row) - def __iter__(self): - return (col for col in self._row) + def __len__(self): + return len(self._row) - def __len__(self): - return len(self._row) + def __repr__(self): + return repr(self._row) - def __repr__(self): - return repr(self._row) + def __getslice__(self, i, j): + return self._row[i,j] - def __getslice__(self, i, j): - return self._row[i,j] + def __getitem__(self, i): + return self._row[i] - def __getitem__(self, i): - return self._row[i] + def __setitem__(self, i, value): + self._row[i] = value - def __setitem__(self, i, value): - self._row[i] = value + def __delitem__(self, i): + del self._row[i] - def __delitem__(self, i): - del self._row[i] + def __getstate__(self): + return {slot: [getattr(self, slot) for slot in self.__slots__]} - def __getstate__(self): - return {slot: getattr(self, slot) for slot in self.__slots__} + def __setstate__(self, state): + for (k, v) in state.items(): setattr(self, k, v) - def __setstate__(self, state): - for (k, v) in state.items(): setattr(self, k, v) + def append(self, value): + self._row.append(value) - def append(self, value): - self._row.append(value) + def insert(self, index, value): + self._row.insert(index, value) - def insert(self, index, value): - self._row.insert(index, value) + def __contains__(self, item): + return (item in self._row) - def __contains__(self, item): - return (item in self._row) + @property + def tuple(self): + '''Tuple representation of :class:`Row`.''' + return tuple(self._row) - @property - def tuple(self): - '''Tuple representation of :class:`Row`.''' - return tuple(self._row) + @property + def list(self): + '''List representation of :class:`Row`.''' + return list(self._row) - @property - def list(self): - '''List representation of :class:`Row`.''' - return list(self._row) + def has_tag(self, tag): + """Returns true if current row contains tag.""" - def has_tag(self, tag): - """Returns true if current row contains tag.""" + if tag == None: + return False + elif isinstance(tag, basestring): + return (tag in self.tags) + else: + return bool(len(set(tag) & set(self.tags))) + - if tag == None: - return False - elif isinstance(tag, basestring): - return (tag in self.tags) - else: - return True if len(set(tag) & set(self.tags)) else False class Dataset(object): - """The :class:`Dataset` object is the heart of Tablib. It provides all core + """The :class:`Dataset` object is the heart of Tablib. It provides all core functionality. Usually you create a :class:`Dataset` instance in your main module, and append @@ -121,346 +128,349 @@ class Dataset(object): defined within the :class:`Dataset` object. To add support for a new format, see :ref:`Adding New Formats <newformats>`. - """ + """ - def __init__(self, *args, **kwargs): - self._data = list(Row(arg) for arg in args) - self.__headers = None + def __init__(self, *args, **kwargs): + self._data = list(Row(arg) for arg in args) + self.__headers = None - # ('title', index) tuples - self._separators = [] + # ('title', index) tuples + self._separators = [] - try: - self.headers = kwargs['headers'] - except KeyError: - self.headers = None + try: + self.headers = kwargs['headers'] + except KeyError: + self.headers = None - try: - self.title = kwargs['title'] - except KeyError: - self.title = None + try: + self.title = kwargs['title'] + except KeyError: + self.title = None - self._register_formats() + self._register_formats() - def __len__(self): - return self.height + def __len__(self): + return self.height - def __getitem__(self, key): - if isinstance(key, basestring): - if key in self.headers: - pos = self.headers.index(key) # get 'key' index from each data - return [row[pos] for row in self._data] - else: - raise KeyError - else: - _results = self._data[key] - if isinstance(_results, Row): - return _results.tuple - else: - return [result.tuple for result in _results] + def __getitem__(self, key): + if isinstance(key, basestring): + if key in self.headers: + pos = self.headers.index(key) # get 'key' index from each data + return [row[pos] for row in self._data] + else: + raise KeyError + else: + _results = self._data[key] + if isinstance(_results, Row): + return _results.tuple + else: + return [result.tuple for result in _results] - def __setitem__(self, key, value): - self._validate(value) - self._data[key] = Row(value) + def __setitem__(self, key, value): + self._validate(value) + self._data[key] = Row(value) - def __delitem__(self, key): - if isinstance(key, basestring): + def __delitem__(self, key): + if isinstance(key, basestring): - if key in self.headers: + if key in self.headers: - pos = self.headers.index(key) - del self.headers[pos] + pos = self.headers.index(key) + del self.headers[pos] - for i, row in enumerate(self._data): + for i, row in enumerate(self._data): - del row[pos] - self._data[i] = row - else: - raise KeyError - else: - del self._data[key] + del row[pos] + self._data[i] = row + else: + raise KeyError + else: + del self._data[key] - def __repr__(self): - try: - return '<%s dataset>' % (self.title.lower()) - except AttributeError: - return '<dataset object>' + def __repr__(self): + try: + return '<%s dataset>' % (self.title.lower()) + except AttributeError: + return '<dataset object>' - @classmethod - def _register_formats(cls): - """Adds format properties.""" - for fmt in formats.available: - try: - try: - setattr(cls, fmt.title, property(fmt.export_set, fmt.import_set)) - except AttributeError: - setattr(cls, fmt.title, property(fmt.export_set)) + @classmethod + def _register_formats(cls): + """Adds format properties.""" + for fmt in formats.available: + try: + try: + setattr(cls, fmt.title, property(fmt.export_set, fmt.import_set)) + except AttributeError: + setattr(cls, fmt.title, property(fmt.export_set)) - except AttributeError: - pass + except AttributeError: + pass - def _validate(self, row=None, col=None, safety=False): - """Assures size of every row in dataset is of proper proportions.""" - if row: - is_valid = (len(row) == self.width) if self.width else True - elif col: - if len(col) < 1: - is_valid = True - else: - is_valid = (len(col) == self.height) if self.height else True - else: - is_valid = all((len(x) == self.width for x in self._data)) + def _validate(self, row=None, col=None, safety=False): + """Assures size of every row in dataset is of proper proportions.""" + if row: + is_valid = (len(row) == self.width) if self.width else True + elif col: + if len(col) < 1: + is_valid = True + else: + is_valid = (len(col) == self.height) if self.height else True + else: + is_valid = all((len(x) == self.width for x in self._data)) - if is_valid: - return True - else: - if not safety: - raise InvalidDimensions - return False + if is_valid: + return True + else: + if not safety: + raise InvalidDimensions + return False - def _package(self, dicts=True): - """Packages Dataset into lists of dictionaries for transmission.""" + def _package(self, dicts=True): + """Packages Dataset into lists of dictionaries for transmission.""" - if self.headers: - if dicts: - data = [dict(zip(self.headers, data_row)) for data_row in self ._data] - else: - data = [list(self.headers)] + list(self._data) - else: - data = [list(row) for row in self._data] + if self.headers: + if dicts: + data = [OrderedDict(zip(self.headers, data_row)) for data_row in self ._data] + else: + data = [list(self.headers)] + list(self._data) + else: + data = [list(row) for row in self._data] - return data + return data - def _clean_col(self, col): - """Prepares the given column for insert/append.""" + def _clean_col(self, col): + """Prepares the given column for insert/append.""" - col = list(col) + col = list(col) - if self.headers: - header = [col.pop(0)] - else: - header = [] + if self.headers: + header = [col.pop(0)] + else: + header = [] - if len(col) == 1 and callable(col[0]): - col = map(col[0], self._data) - col = tuple(header + col) + if len(col) == 1 and callable(col[0]): + col = map(col[0], self._data) + col = tuple(header + col) - return col + return col - @property - def height(self): - """The number of rows currently in the :class:`Dataset`. - Cannot be directly modified. - """ - return len(self._data) + @property + def height(self): + """The number of rows currently in the :class:`Dataset`. + Cannot be directly modified. + """ + return len(self._data) - @property - def width(self): - """The number of columns currently in the :class:`Dataset`. - Cannot be directly modified. - """ + @property + def width(self): + """The number of columns currently in the :class:`Dataset`. + Cannot be directly modified. + """ - try: - return len(self._data[0]) - except IndexError: - try: - return len(self.headers) - except TypeError: - return 0 + try: + return len(self._data[0]) + except IndexError: + try: + return len(self.headers) + except TypeError: + return 0 - @property - def headers(self): - """An *optional* list of strings to be used for header rows and attribute names. + def _get_headers(self): + """An *optional* list of strings to be used for header rows and attribute names. This must be set manually. The given list length must equal :class:`Dataset.width`. - """ - return self.__headers + """ + return self.__headers - @headers.setter - def headers(self, collection): - """Validating headers setter.""" - self._validate(collection) - if collection: - try: - self.__headers = list(collection) - except TypeError: - raise TypeError - else: - self.__headers = None + def _set_headers(self, collection): + """Validating headers setter.""" + self._validate(collection) + if collection: + try: + self.__headers = list(collection) + except TypeError: + raise TypeError + else: + self.__headers = None + headers = property(_get_headers, _set_headers) - @property - def dict(self): - """A JSON representation of the :class:`Dataset` object. If headers have been - set, a JSON list of objects will be returned. If no headers have - been set, a JSON list of lists (rows) will be returned instead. + def _get_dict(self): + """A native Python representation of the :class:`Dataset` object. If headers have + been set, a list of Python dictionaries will be returned. If no headers have been set, + a list of tuples (rows) will be returned instead. - A dataset object can also be imported by setting the `Dataset.json` attribute: :: + A dataset object can also be imported by setting the `Dataset.dict` attribute: :: data = tablib.Dataset() data.json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]' - """ - return self._package() + """ + return self._package() - @dict.setter - def dict(self, pickle): - """A native Python representation of the Dataset object. If headers have been - set, a list of Python dictionaries will be returned. If no headers have been - set, a list of tuples (rows) will be returned instead. + def _set_dict(self, pickle): + """A native Python representation of the Dataset object. If headers have been + set, a list of Python dictionaries will be returned. If no headers have been + set, a list of tuples (rows) will be returned instead. - A dataset object can also be imported by setting the :class:`Dataset.dict` attribute. :: + A dataset object can also be imported by setting the :class:`Dataset.dict` attribute. :: - data = tablib.Dataset() - data.dict = [{'age': 90, 'first_name': 'Kenneth', 'last_name': 'Reitz'}] + data = tablib.Dataset() + data.dict = [{'age': 90, 'first_name': 'Kenneth', 'last_name': 'Reitz'}] + + """ + + if not len(pickle): + return - """ - if not len(pickle): - return + # if list of rows + if isinstance(pickle[0], list): + self.wipe() + for row in pickle: + self.append(Row(row)) - # if list of rows - if isinstance(pickle[0], list): - self.wipe() - for row in pickle: - self.append(Row(row)) + # if list of objects + elif isinstance(pickle[0], dict): + self.wipe() + self.headers = pickle[0].keys() + for row in pickle: + self.append(Row(row.values())) + else: + raise UnsupportedFormat - # if list of objects - elif isinstance(pickle[0], dict): - self.wipe() - self.headers = pickle[0].keys() - for row in pickle: - self.append(Row(row.values())) - else: - raise UnsupportedFormat + dict = property(_get_dict, _set_dict) - @property - def xls(): - """An Excel Spreadsheet representation of the :class:`Dataset` object, with :ref:`seperators`. Cannot be set. - .. admonition:: Binary Warning + @property + def xls(): + """An Excel Spreadsheet representation of the :class:`Dataset` object, with :ref:`seperators`. Cannot be set. - :class:`Dataset.xls` contains binary data, so make sure to write in binary mode:: + .. admonition:: Binary Warning + + :class:`Dataset.xls` contains binary data, so make sure to write in binary mode:: + + with open('output.xls', 'wb') as f: + f.write(data.xls)' + """ + pass - with open('output.xls', 'wb') as f: - f.write(data.xls)' - """ - pass + @property + def csv(): + """A CSV representation of the :class:`Dataset` object. The top row will contain + headers, if they have been set. Otherwise, the top row will contain + the first row of the dataset. - @property - def csv(): - """A CSV representation of the :class:`Dataset` object. The top row will contain - headers, if they have been set. Otherwise, the top row will contain - the first row of the dataset. + A dataset object can also be imported by setting the :class:`Dataset.csv` attribute. :: - A dataset object can also be imported by setting the :class:`Dataset.csv` attribute. :: + data = tablib.Dataset() + data.csv = 'age, first_name, last_name\\n90, John, Adams' - data = tablib.Dataset() - data.csv = 'age, first_name, last_name\\n90, John, Adams' + Import assumes (for now) that headers exist. + """ + pass - Import assumes (for now) that headers exist. - """ - pass + @property + def tsv(): + """A TSV representation of the :class:`Dataset` object. The top row will contain + headers, if they have been set. Otherwise, the top row will contain + the first row of the dataset. - @property - def tsv(): - """A TSV representation of the :class:`Dataset` object. The top row will contain - headers, if they have been set. Otherwise, the top row will contain - the first row of the dataset. + A dataset object can also be imported by setting the :class:`Dataset.tsv` attribute. :: - A dataset object can also be imported by setting the :class:`Dataset.tsv` attribute. :: + data = tablib.Dataset() + data.tsv = 'age\tfirst_name\tlast_name\\n90\tJohn\tAdams' - data = tablib.Dataset() - data.tsv = 'age\tfirst_name\tlast_name\\n90\tJohn\tAdams' + Import assumes (for now) that headers exist. + """ - Import assumes (for now) that headers exist. - """ + @property + def yaml(): + """A YAML representation of the :class:`Dataset` object. If headers have been + set, a YAML list of objects will be returned. If no headers have + been set, a YAML list of lists (rows) will be returned instead. - @property - def yaml(): - """A YAML representation of the :class:`Dataset` object. If headers have been - set, a YAML list of objects will be returned. If no headers have - been set, a YAML list of lists (rows) will be returned instead. + A dataset object can also be imported by setting the :class:`Dataset.json` attribute: :: - A dataset object can also be imported by setting the :class:`Dataset.json` attribute: :: + data = tablib.Dataset() + data.yaml = '- {age: 90, first_name: John, last_name: Adams}' - data = tablib.Dataset() - data.yaml = '- {age: 90, first_name: John, last_name: Adams}' + Import assumes (for now) that headers exist. + """ + pass - Import assumes (for now) that headers exist. - """ - pass + @property + def json(): + """A JSON representation of the :class:`Dataset` object. If headers have been + set, a JSON list of objects will be returned. If no headers have + been set, a JSON list of lists (rows) will be returned instead. - @property - def json(): - """A JSON representation of the :class:`Dataset` object. If headers have been - set, a JSON list of objects will be returned. If no headers have - been set, a JSON list of lists (rows) will be returned instead. + A dataset object can also be imported by setting the :class:`Dataset.json` attribute: :: - A dataset object can also be imported by setting the :class:`Dataset.json` attribute: :: + data = tablib.Dataset() + data.json = '[{age: 90, first_name: "John", liast_name: "Adams"}]' - data = tablib.Dataset() - data.json = '[{age: 90, first_name: "John", liast_name: "Adams"}]' + Import assumes (for now) that headers exist. + """ - Import assumes (for now) that headers exist. - """ + @property + def html(): + """A HTML table representation of the :class:`Dataset` object. If + headers have been set, they will be used as table headers. - @property - def html(): - """A HTML table representation of the :class:`Dataset` object. If - headers have been set, they will be used as table headers. + ..notice:: This method can be used for export only. + """ + pass - ..notice:: This method can be used for export only. - """ - pass - def append(self, row=None, col=None, header=None, tags=list()): - """Adds a row or column to the :class:`Dataset`. - Usage is :class:`Dataset.insert` for documentation. + def append(self, row=None, col=None, header=None, tags=list()): + """Adds a row or column to the :class:`Dataset`. + Usage is :class:`Dataset.insert` for documentation. """ - if row is not None: - self.insert(self.height, row=row, tags=tags) - elif col is not None: - self.insert(self.width, col=col, header=header) + if row is not None: + self.insert(self.height, row=row, tags=tags) + elif col is not None: + self.insert(self.width, col=col, header=header) + - def insert_separator(self, index, text='-'): - """Adds a separator to :class:`Dataset` at given index.""" + def insert_separator(self, index, text='-'): + """Adds a separator to :class:`Dataset` at given index.""" - sep = (index, text) - self._separators.append(sep) + sep = (index, text) + self._separators.append(sep) - def append_separator(self, text='-'): - """Adds a :ref:`seperator <seperators>` to the :class:`Dataset`.""" + def append_separator(self, text='-'): + """Adds a :ref:`seperator <seperators>` to the :class:`Dataset`.""" - # change offsets if headers are or aren't defined - if not self.headers: - index = self.height if self.height else 0 - else: - index = (self.height + 1) if self.height else 1 + # change offsets if headers are or aren't defined + if not self.headers: + index = self.height if self.height else 0 + else: + index = (self.height + 1) if self.height else 1 - self.insert_separator(index, text) + self.insert_separator(index, text) - def insert(self, index, row=None, col=None, header=None, tags=list()): - """Inserts a row or column to the :class:`Dataset` at the given index. + def insert(self, index, row=None, col=None, header=None, tags=list()): + """Inserts a row or column to the :class:`Dataset` at the given index. Rows and columns inserted must be the correct size (height or width). @@ -486,270 +496,271 @@ class Dataset(object): This gives you the ability to :class:`filter <Dataset.filter>` your :class:`Dataset` later. - """ - if row: - self._validate(row) - self._data.insert(index, Row(row, tags=tags)) - elif col: - col = list(col) + """ + if row: + self._validate(row) + self._data.insert(index, Row(row, tags=tags)) + elif col: + col = list(col) + + # Callable Columns... + if len(col) == 1 and callable(col[0]): + col = map(col[0], self._data) - # Callable Columns... - if len(col) == 1 and callable(col[0]): - col = map(col[0], self._data) + col = self._clean_col(col) + self._validate(col=col) - col = self._clean_col(col) - self._validate(col=col) + if self.headers: + # pop the first item off, add to headers + if not header: + raise HeadersNeeded() + self.headers.insert(index, header) - if self.headers: - # pop the first item off, add to headers - if not header: - raise HeadersNeeded() - self.headers.insert(index, header) + if self.height and self.width: - if self.height and self.width: + for i, row in enumerate(self._data): - for i, row in enumerate(self._data): + row.insert(index, col[i]) + self._data[i] = row + else: + self._data = [Row([row]) for row in col] - row.insert(index, col[i]) - self._data[i] = row - else: - self._data = [Row([row]) for row in col] + def filter(self, tag): + """Returns a new instance of the :class:`Dataset`, excluding any rows + that do not contain the given :ref:`tags <tags>`. + """ + _dset = copy(self) + _dset._data = [row for row in _dset._data if row.has_tag(tag)] - def filter(self, tag): - """Returns a new instance of the :class:`Dataset`, excluding any rows - that do not contain the given :ref:`tags <tags>`. - """ - _dset = copy(self) - _dset._data = [row for row in _dset._data if row.has_tag(tag)] + return _dset - return _dset + def sort(self, col, reverse=False): + """Sort a :class:`Dataset` by a specific column, given string (for + header) or integer (for column index). The order can be reversed by + setting ``reverse`` to ``True``. + Returns a new :class:`Dataset` instance where columns have been + sorted.""" + + if isinstance(col, basestring): - def sort(self, col, reverse=False): - """Sort a :class:`Dataset` by a specific column, given string (for - header) or integer (for column index). The order can be reversed by - setting ``reverse`` to ``True``. - Returns a new :class:`Dataset` instance where columns have been - sorted.""" - - if isinstance(col, basestring): + if not self.headers: + raise HeadersNeeded - if not self.headers: - raise HeadersNeeded + _sorted = sorted(self.dict, key=itemgetter(col), reverse=reverse) + _dset = Dataset(headers=self.headers) - _sorted = sorted(self.dict, key=itemgetter(col), reverse=reverse) - _dset = Dataset(headers=self.headers) + for item in _sorted: + row = [item[key] for key in self.headers] + _dset.append(row=row) - for item in _sorted: - row = [item[key] for key in self.headers] - _dset.append(row=row) + else: + if self.headers: + col = self.headers[col] - else: - if self.headers: - col = self.headers[col] + _sorted = sorted(self.dict, key=itemgetter(col), reverse=reverse) + _dset = Dataset(headers=self.headers) - _sorted = sorted(self.dict, key=itemgetter(col), reverse=reverse) - _dset = Dataset(headers=self.headers) + for item in _sorted: + if self.headers: + row = [item[key] for key in self.headers] + else: + row = item + _dset.append(row=row) - for item in _sorted: - if self.headers: - row = [item[key] for key in self.headers] - else: - row = item - _dset.append(row=row) + return _dset - return _dset - def transpose(self): - """Transpose a :class:`Dataset`, turning rows into columns and vice - versa, returning a new ``Dataset`` instance. The first row of the - original instance becomes the new header row.""" + def transpose(self): + """Transpose a :class:`Dataset`, turning rows into columns and vice + versa, returning a new ``Dataset`` instance. The first row of the + original instance becomes the new header row.""" - # Don't transpose if there is no data - if not self: - return + # Don't transpose if there is no data + if not self: + return - _dset = Dataset() - # The first element of the headers stays in the headers, - # it is our "hinge" on which we rotate the data - new_headers = [self.headers[0]] + self[self.headers[0]] + _dset = Dataset() + # The first element of the headers stays in the headers, + # it is our "hinge" on which we rotate the data + new_headers = [self.headers[0]] + self[self.headers[0]] - _dset.headers = new_headers - for column in self.headers: + _dset.headers = new_headers + for column in self.headers: - if column == self.headers[0]: - # It's in the headers, so skip it - continue + if column == self.headers[0]: + # It's in the headers, so skip it + continue - # Adding the column name as now they're a regular column - row_data = [column] + self[column] - row_data = Row(row_data) - _dset.append(row=row_data) + # Adding the column name as now they're a regular column + row_data = [column] + self[column] + row_data = Row(row_data) + _dset.append(row=row_data) - return _dset + return _dset - def stack_rows(self, other): - """Stack two :class:`Dataset` instances together by - joining at the row level, and return new combined - ``Dataset`` instance.""" + def stack_rows(self, other): + """Stack two :class:`Dataset` instances together by + joining at the row level, and return new combined + ``Dataset`` instance.""" - if not isinstance(other, Dataset): - return + if not isinstance(other, Dataset): + return - if self.width != other.width: - raise InvalidDimensions + if self.width != other.width: + raise InvalidDimensions - # Copy the source data - _dset = copy(self) + # Copy the source data + _dset = copy(self) - rows_to_stack = [row for row in _dset._data] - other_rows = [row for row in other._data] + rows_to_stack = [row for row in _dset._data] + other_rows = [row for row in other._data] - rows_to_stack.extend(other_rows) - _dset._data = rows_to_stack + rows_to_stack.extend(other_rows) + _dset._data = rows_to_stack - return _dset + return _dset - def stack_columns(self, other): - """Stack two :class:`Dataset` instances together by - joining at the column level, and return a new - combined ``Dataset`` instance. If either ``Dataset`` - has headers set, than the other must as well.""" + def stack_columns(self, other): + """Stack two :class:`Dataset` instances together by + joining at the column level, and return a new + combined ``Dataset`` instance. If either ``Dataset`` + has headers set, than the other must as well.""" - if not isinstance(other, Dataset): - return + if not isinstance(other, Dataset): + return - if self.headers or other.headers: - if not self.headers or not other.headers: - raise HeadersNeeded + if self.headers or other.headers: + if not self.headers or not other.headers: + raise HeadersNeeded - if self.height != other.height: - raise InvalidDimensions + if self.height != other.height: + raise InvalidDimensions - try: - new_headers = self.headers + other.headers - except TypeError: - new_headers = None + try: + new_headers = self.headers + other.headers + except TypeError: + new_headers = None - _dset = Dataset() + _dset = Dataset() - for column in self.headers: - _dset.append(col=self[column]) + for column in self.headers: + _dset.append(col=self[column]) - for column in other.headers: - _dset.append(col=other[column]) + for column in other.headers: + _dset.append(col=other[column]) - _dset.headers = new_headers + _dset.headers = new_headers - return _dset + return _dset - def wipe(self): - """Removes all content and headers from the :class:`Dataset` object.""" - self._data = list() - self.__headers = None + def wipe(self): + """Removes all content and headers from the :class:`Dataset` object.""" + self._data = list() + self.__headers = None class Databook(object): - """A book of :class:`Dataset` objects. - """ + """A book of :class:`Dataset` objects. + """ - def __init__(self, sets=None): + def __init__(self, sets=None): - if sets is None: - self._datasets = list() - else: - self._datasets = sets + if sets is None: + self._datasets = list() + else: + self._datasets = sets - self._register_formats() + self._register_formats() - def __repr__(self): - try: - return '<%s databook>' % (self.title.lower()) - except AttributeError: - return '<databook object>' + def __repr__(self): + try: + return '<%s databook>' % (self.title.lower()) + except AttributeError: + return '<databook object>' - def wipe(self): - """Removes all :class:`Dataset` objects from the :class:`Databook`.""" - self._datasets = [] + def wipe(self): + """Removes all :class:`Dataset` objects from the :class:`Databook`.""" + self._datasets = [] - @classmethod - def _register_formats(cls): - """Adds format properties.""" - for fmt in formats.available: - try: - try: - setattr(cls, fmt.title, property(fmt.export_book, fmt.import_book)) - except AttributeError: - setattr(cls, fmt.title, property(fmt.export_book)) + @classmethod + def _register_formats(cls): + """Adds format properties.""" + for fmt in formats.available: + try: + try: + setattr(cls, fmt.title, property(fmt.export_book, fmt.import_book)) + except AttributeError: + setattr(cls, fmt.title, property(fmt.export_book)) - except AttributeError: - pass + except AttributeError: + pass - def add_sheet(self, dataset): - """Adds given :class:`Dataset` to the :class:`Databook`.""" - if type(dataset) is Dataset: - self._datasets.append(dataset) - else: - raise InvalidDatasetType + def add_sheet(self, dataset): + """Adds given :class:`Dataset` to the :class:`Databook`.""" + if type(dataset) is Dataset: + self._datasets.append(dataset) + else: + raise InvalidDatasetType - def _package(self): - """Packages :class:`Databook` for delivery.""" - collector = [] - for dset in self._datasets: - collector.append(dict( - title = dset.title, - data = dset.dict - )) - return collector + def _package(self): + """Packages :class:`Databook` for delivery.""" + collector = [] + for dset in self._datasets: + collector.append(OrderedDict( + title = dset.title, + data = dset.dict + )) + return collector - @property - def size(self): - """The number of the :class:`Dataset` objects within :class:`Databook`.""" - return len(self._datasets) + @property + def size(self): + """The number of the :class:`Dataset` objects within :class:`Databook`.""" + return len(self._datasets) def detect(stream): - """Return (format, stream) of given stream.""" - for fmt in formats.available: - try: - if fmt.detect(stream): - return (fmt, stream) - except AttributeError: - pass - return (None, stream) + """Return (format, stream) of given stream.""" + for fmt in formats.available: + try: + if fmt.detect(stream): + return (fmt, stream) + except AttributeError: + pass + return (None, stream) def import_set(stream): - """Return dataset of given stream.""" - (format, stream) = detect(stream) + """Return dataset of given stream.""" + (format, stream) = detect(stream) - try: - data = Dataset() - format.import_set(data, stream) - return data + try: + data = Dataset() + format.import_set(data, stream) + return data - except AttributeError, e: - return None + except AttributeError, e: + return None class InvalidDatasetType(Exception): - "Only Datasets can be added to a DataBook" + "Only Datasets can be added to a DataBook" class InvalidDimensions(Exception): - "Invalid size" + "Invalid size" class HeadersNeeded(Exception): - "Header parameter must be given when appending a column in this Dataset." + "Header parameter must be given when appending a column in this Dataset." class UnsupportedFormat(NotImplementedError): - "Format is not supported" + "Format is not supported" diff --git a/tablib/formats/_csv.py b/tablib/formats/_csv.py index 2391417..2c74a1c 100644 --- a/tablib/formats/_csv.py +++ b/tablib/formats/_csv.py @@ -16,34 +16,34 @@ extentions = ('csv',) def export_set(dataset): - """Returns CSV representation of Dataset.""" - stream = cStringIO.StringIO() - _csv = csv.writer(stream) + """Returns CSV representation of Dataset.""" + stream = cStringIO.StringIO() + _csv = csv.writer(stream) - for row in dataset._package(dicts=False): - _csv.writerow(row) + for row in dataset._package(dicts=False): + _csv.writerow(row) - return stream.getvalue() + return stream.getvalue() def import_set(dset, in_stream, headers=True): - """Returns dataset from CSV stream.""" + """Returns dataset from CSV stream.""" - dset.wipe() + dset.wipe() - rows = csv.reader(in_stream.split()) - for i, row in enumerate(rows): + rows = csv.reader(in_stream.split()) + for i, row in enumerate(rows): - if (i == 0) and (headers): - dset.headers = row - else: - dset.append(row) + if (i == 0) and (headers): + dset.headers = row + else: + dset.append(row) def detect(stream): - """Returns True if given stream is valid CSV.""" - try: - rows = dialect = csv.Sniffer().sniff(stream) - return True - except csv.Error: - return False
\ No newline at end of file + """Returns True if given stream is valid CSV.""" + try: + rows = dialect = csv.Sniffer().sniff(stream) + return True + except csv.Error: + return False
\ No newline at end of file diff --git a/tablib/formats/_json.py b/tablib/formats/_json.py index 7f31ee5..262c627 100644 --- a/tablib/formats/_json.py +++ b/tablib/formats/_json.py @@ -3,15 +3,9 @@ """ Tablib - JSON Support """ -try: - import json # load system JSON (Python >= 2.6) -except ImportError: - try: - import simplejson as json - except ImportError: - import tablib.packages.simplejson as json # use the vendorized copy - import tablib.core +from tablib.packages import anyjson + title = 'json' @@ -19,37 +13,37 @@ extentions = ('json', 'jsn') def export_set(dataset): - """Returns JSON representation of Dataset.""" - return json.dumps(dataset.dict) + """Returns JSON representation of Dataset.""" + return anyjson.serialize(dataset.dict) def export_book(databook): - """Returns JSON representation of Databook.""" - return json.dumps(databook._package()) + """Returns JSON representation of Databook.""" + return anyjson.serialize(databook._package()) def import_set(dset, in_stream): - """Returns dataset from JSON stream.""" + """Returns dataset from JSON stream.""" - dset.wipe() - dset.dict = json.loads(in_stream) + dset.wipe() + dset.dict = anyjson.deserialize(in_stream) def import_book(dbook, in_stream): - """Returns databook from JSON stream.""" + """Returns databook from JSON stream.""" - dbook.wipe() - for sheet in json.loads(in_stream): - data = tablib.core.Dataset() - data.title = sheet['title'] - data.dict = sheet['data'] - dbook.add_sheet(data) + dbook.wipe() + for sheet in anyjson.deserialize(in_stream): + data = tablib.core.Dataset() + data.title = sheet['title'] + data.dict = sheet['data'] + dbook.add_sheet(data) def detect(stream): - """Returns True if given stream is valid JSON.""" - try: - json.loads(stream) - return True - except ValueError: - return False + """Returns True if given stream is valid JSON.""" + try: + anyjson.deserialize(stream) + return True + except ValueError: + return False diff --git a/tablib/formats/_tsv.py b/tablib/formats/_tsv.py index 808f202..76a5f07 100644 --- a/tablib/formats/_tsv.py +++ b/tablib/formats/_tsv.py @@ -16,36 +16,36 @@ extentions = ('tsv',) def export_set(dataset): - """Returns a TSV representation of Dataset.""" - stream = cStringIO.StringIO() - _tsv = csv.writer(stream, delimiter='\t') + """Returns a TSV representation of Dataset.""" + stream = cStringIO.StringIO() + _tsv = csv.writer(stream, delimiter='\t') - for row in dataset._package(dicts=False): - _tsv.writerow(row) + for row in dataset._package(dicts=False): + _tsv.writerow(row) - return stream.getvalue() + return stream.getvalue() def import_set(dset, in_stream, headers=True): - """Returns dataset from TSV stream.""" - dset.wipe() + """Returns dataset from TSV stream.""" + dset.wipe() - rows = csv.reader(in_stream.split('\r\n'), delimiter='\t') - for i, row in enumerate(rows): - # Skip empty rows - if not row: - continue + rows = csv.reader(in_stream.split('\r\n'), delimiter='\t') + for i, row in enumerate(rows): + # Skip empty rows + if not row: + continue - if (i == 0) and (headers): - dset.headers = row - else: - dset.append(row) + if (i == 0) and (headers): + dset.headers = row + else: + dset.append(row) def detect(stream): - """Returns True if given stream is valid TSV.""" - try: - rows = dialect = csv.Sniffer().sniff(stream, delimiters='\t') - return True - except csv.Error: - return False + """Returns True if given stream is valid TSV.""" + try: + rows = dialect = csv.Sniffer().sniff(stream, delimiters='\t') + return True + except csv.Error: + return False diff --git a/tablib/formats/_xls.py b/tablib/formats/_xls.py index 08bc0f6..717a6d5 100644 --- a/tablib/formats/_xls.py +++ b/tablib/formats/_xls.py @@ -6,9 +6,9 @@ import cStringIO try: - import xlwt + import xlwt except ImportError: - import tablib.packages.xlwt as xlwt + import tablib.packages.xlwt as xlwt title = 'xls' @@ -20,66 +20,66 @@ bold = xlwt.easyxf("font: bold on") def export_set(dataset): - """Returns XLS representation of Dataset.""" + """Returns XLS representation of Dataset.""" - wb = xlwt.Workbook(encoding='utf8') - ws = wb.add_sheet(dataset.title if dataset.title else 'Tabbed Dataset') + wb = xlwt.Workbook(encoding='utf8') + ws = wb.add_sheet(dataset.title if dataset.title else 'Tabbed Dataset') - dset_sheet(dataset, ws) + dset_sheet(dataset, ws) - stream = cStringIO.StringIO() - wb.save(stream) - return stream.getvalue() + stream = cStringIO.StringIO() + wb.save(stream) + return stream.getvalue() def export_book(databook): - """Returns XLS representation of DataBook.""" + """Returns XLS representation of DataBook.""" - wb = xlwt.Workbook(encoding='utf8') + wb = xlwt.Workbook(encoding='utf8') - for i, dset in enumerate(databook._datasets): - ws = wb.add_sheet(dset.title if dset.title else 'Sheet%s' % (i)) + for i, dset in enumerate(databook._datasets): + ws = wb.add_sheet(dset.title if dset.title else 'Sheet%s' % (i)) - dset_sheet(dset, ws) + dset_sheet(dset, ws) - stream = cStringIO.StringIO() - wb.save(stream) - return stream.getvalue() + stream = cStringIO.StringIO() + wb.save(stream) + return stream.getvalue() def dset_sheet(dataset, ws): - """Completes given worksheet from given Dataset.""" - _package = dataset._package(dicts=False) - - for i, sep in enumerate(dataset._separators): - _offset = i - _package.insert((sep[0] + _offset), (sep[1],)) - - for i, row in enumerate(_package): - for j, col in enumerate(row): - - # bold headers - if (i == 0) and dataset.headers: - ws.write(i, j, col, bold) - - # frozen header row - ws.panes_frozen = True - ws.horz_split_pos = 1 - - - # bold separators - elif len(row) < dataset.width: - ws.write(i, j, col, bold) - - # wrap the rest - else: - try: - if '\n' in col: - ws.write(i, j, col, wrap) - else: - ws.write(i, j, col) - except TypeError: - ws.write(i, j, col) + """Completes given worksheet from given Dataset.""" + _package = dataset._package(dicts=False) + + for i, sep in enumerate(dataset._separators): + _offset = i + _package.insert((sep[0] + _offset), (sep[1],)) + + for i, row in enumerate(_package): + for j, col in enumerate(row): + + # bold headers + if (i == 0) and dataset.headers: + ws.write(i, j, col, bold) + + # frozen header row + ws.panes_frozen = True + ws.horz_split_pos = 1 + + + # bold separators + elif len(row) < dataset.width: + ws.write(i, j, col, bold) + + # wrap the rest + else: + try: + if '\n' in col: + ws.write(i, j, col, wrap) + else: + ws.write(i, j, col) + except TypeError: + ws.write(i, j, col) diff --git a/tablib/formats/_yaml.py b/tablib/formats/_yaml.py index 59d49a0..3f2f8b7 100644 --- a/tablib/formats/_yaml.py +++ b/tablib/formats/_yaml.py @@ -4,9 +4,9 @@ """ try: - import yaml + import yaml except ImportError: - import tablib.packages.yaml as yaml + import tablib.packages.yaml as yaml import tablib @@ -18,40 +18,40 @@ extentions = ('yaml', 'yml') def export_set(dataset): - """Returns YAML representation of Dataset.""" - return yaml.dump(dataset.dict) + """Returns YAML representation of Dataset.""" + return yaml.dump(dataset.dict) def export_book(databook): - """Returns YAML representation of Databook.""" - return yaml.dump(databook._package()) + """Returns YAML representation of Databook.""" + return yaml.dump(databook._package()) def import_set(dset, in_stream): - """Returns dataset from YAML stream.""" + """Returns dataset from YAML stream.""" - dset.wipe() - dset.dict = yaml.load(in_stream) + dset.wipe() + dset.dict = yaml.load(in_stream) def import_book(dbook, in_stream): - """Returns databook from YAML stream.""" + """Returns databook from YAML stream.""" - dbook.wipe() + dbook.wipe() - for sheet in yaml.load(in_stream): - data = tablib.core.Dataset() - data.title = sheet['title'] - data.dict = sheet['data'] - dbook.add_sheet(data) - + for sheet in yaml.load(in_stream): + data = tablib.core.Dataset() + data.title = sheet['title'] + data.dict = sheet['data'] + dbook.add_sheet(data) + def detect(stream): - """Returns True if given stream is valid YAML.""" - try: - _yaml = yaml.load(stream) - if isinstance(_yaml, (list, tuple, dict)): - return True - else: - return False - except yaml.parser.ParserError: - return False
\ No newline at end of file + """Returns True if given stream is valid YAML.""" + try: + _yaml = yaml.load(stream) + if isinstance(_yaml, (list, tuple, dict)): + return True + else: + return False + except yaml.parser.ParserError: + return False
\ No newline at end of file diff --git a/tablib/helpers.py b/tablib/helpers.py index f75c466..718d159 100644 --- a/tablib/helpers.py +++ b/tablib/helpers.py @@ -7,31 +7,31 @@ import sys class Struct(object): - """Your attributes are belong to us.""" - - def __init__(self, **entries): - self.__dict__.update(entries) - - def __getitem__(self, key): - return getattr(self, key, None) + """Your attributes are belong to us.""" + + def __init__(self, **entries): + self.__dict__.update(entries) + + def __getitem__(self, key): + return getattr(self, key, None) - def dictionary(self): - """Returns dictionary representation of object.""" - return self.__dict__ + def dictionary(self): + """Returns dictionary representation of object.""" + return self.__dict__ - def items(self): - """Returns items within object.""" - return self.__dict__.items() + def items(self): + """Returns items within object.""" + return self.__dict__.items() - def keys(self): - """Returns keys within object.""" - return self.__dict__.keys() + def keys(self): + """Returns keys within object.""" + return self.__dict__.keys() def piped(): - """Returns piped input via stdin, else False.""" - with sys.stdin as stdin: - # TTY is only way to detect if stdin contains data - return stdin.read() if not stdin.isatty() else None + """Returns piped input via stdin, else False.""" + with sys.stdin as stdin: + # TTY is only way to detect if stdin contains data + return stdin.read() if not stdin.isatty() else None diff --git a/tablib/packages/anyjson.py b/tablib/packages/anyjson.py new file mode 100644 index 0000000..6603751 --- /dev/null +++ b/tablib/packages/anyjson.py @@ -0,0 +1,117 @@ +""" +Wraps the best available JSON implementation available in a common interface +""" + +__version__ = "0.2.0" +__author__ = "Rune Halvorsen <runefh@gmail.com>" +__homepage__ = "http://bitbucket.org/runeh/anyjson/" +__docformat__ = "restructuredtext" + +""" + +.. function:: serialize(obj) + + Serialize the object to JSON. + +.. function:: deserialize(str) + + Deserialize JSON-encoded object to a Python object. + +.. function:: force_implementation(name) + + Load a specific json module. This is useful for testing and not much else + +.. attribute:: implementation + + The json implementation object. This is probably not useful to you, + except to get the name of the implementation in use. The name is + available through `implementation.name`. +""" + +import sys + +implementation = None + +""" +.. data:: _modules + + List of known json modules, and the names of their serialize/unserialize + methods, as well as the exception they throw. Exception can be either + an exception class or a string. +""" +_modules = [("cjson", "encode", "EncodeError", "decode", "DecodeError"), + ("jsonlib2", "write", "WriteError", "read", "ReadError"), + ("jsonlib", "write", "WriteError", "read", "ReadError"), + ("simplejson", "dumps", TypeError, "loads", ValueError), + ("json", "dumps", TypeError, "loads", ValueError), + ("django.utils.simplejson", "dumps", TypeError, "loads", + ValueError)] +_fields = ("modname", "encoder", "encerror", "decoder", "decerror") + + +class _JsonImplementation(object): + """Incapsulates a JSON implementation""" + + def __init__(self, modspec): + modinfo = dict(zip(_fields, modspec)) + + # No try block. We want importerror to end up at caller + module = self._attempt_load(modinfo["modname"]) + + self.implementation = modinfo["modname"] + self._encode = getattr(module, modinfo["encoder"]) + self._decode = getattr(module, modinfo["decoder"]) + self._encode_error = modinfo["encerror"] + self._decode_error = modinfo["decerror"] + + if isinstance(modinfo["encerror"], basestring): + self._encode_error = getattr(module, modinfo["encerror"]) + if isinstance(modinfo["decerror"], basestring): + self._decode_error = getattr(module, modinfo["decerror"]) + + self.name = modinfo["modname"] + + def _attempt_load(self, modname): + """Attempt to load module name modname, returning it on success, + throwing ImportError if module couldn't be imported""" + __import__(modname) + return sys.modules[modname] + + def serialize(self, data): + """Serialize the datastructure to json. Returns a string. Raises + TypeError if the object could not be serialized.""" + try: + return self._encode(data) + except self._encode_error, exc: + raise TypeError(*exc.args) + + def deserialize(self, s): + """deserialize the string to python data types. Raises + ValueError if the string vould not be parsed.""" + try: + return self._decode(s) + except self._decode_error, exc: + raise ValueError(*exc.args) + + +def force_implementation(modname): + """Forces anyjson to use a specific json module if it's available""" + global implementation + for name, spec in [(e[0], e) for e in _modules]: + if name == modname: + implementation = _JsonImplementation(spec) + return + raise ImportError("No module named: %s" % modname) + + +for modspec in _modules: + try: + implementation = _JsonImplementation(modspec) + break + except ImportError: + pass +else: + raise ImportError("No supported JSON module found") + +serialize = lambda value: implementation.serialize(value) +deserialize = lambda value: implementation.deserialize(value) diff --git a/tablib/packages/markup.py b/tablib/packages/markup.py index 98d9b1d..234f116 100644 --- a/tablib/packages/markup.py +++ b/tablib/packages/markup.py @@ -29,10 +29,10 @@ class element: def __init__( self, tag, case='lower', parent=None ): self.parent = parent - if case == 'lower': - self.tag = tag.lower( ) - else: - self.tag = tag.upper( ) + if case == 'lower': + self.tag = tag.lower( ) + else: + self.tag = tag.upper( ) def __call__( self, *args, **kwargs ): if len( args ) > 1: @@ -67,8 +67,8 @@ class element: def render( self, tag, single, between, kwargs ): """Append the actual tags to content.""" - out = "<%s" % tag - for key, value in kwargs.iteritems( ): + out = "<%s" % tag + for key, value in kwargs.iteritems( ): if value is not None: # when value is None that means stuff like <... checked> key = key.strip('_') # strip this so class_ will mean class, etc. if key == 'http_equiv': # special cases, maybe change _ to - overall? @@ -78,13 +78,13 @@ class element: out = "%s %s=\"%s\"" % ( out, key, escape( value ) ) else: out = "%s %s" % ( out, key ) - if between is not None: - out = "%s>%s</%s>" % ( out, between, tag ) - else: - if single: - out = "%s />" % out - else: - out = "%s>" % out + if between is not None: + out = "%s>%s</%s>" % ( out, between, tag ) + else: + if single: + out = "%s />" % out + else: + out = "%s>" % out if self.parent is not None: self.parent.content.append( out ) else: @@ -145,30 +145,30 @@ class page: deprecated_twotags = [ "APPLET", "CENTER", "DIR", "FONT", "MENU", "S", "STRIKE", "U" ] self.header = [ ] - self.content = [ ] + self.content = [ ] self.footer = [ ] - self.case = case + self.case = case self.separator = separator # init( ) sets it to True so we know that </body></html> has to be printed at the end self._full = False self.class_= class_ - if mode == 'strict_html' or mode == 'html': - self.onetags = valid_onetags - self.onetags += map( string.lower, self.onetags ) - self.twotags = valid_twotags - self.twotags += map( string.lower, self.twotags ) - self.deptags = deprecated_onetags + deprecated_twotags - self.deptags += map( string.lower, self.deptags ) - self.mode = 'strict_html' - elif mode == 'loose_html': - self.onetags = valid_onetags + deprecated_onetags - self.onetags += map( string.lower, self.onetags ) - self.twotags = valid_twotags + deprecated_twotags - self.twotags += map( string.lower, self.twotags ) - self.mode = mode - elif mode == 'xml': + if mode == 'strict_html' or mode == 'html': + self.onetags = valid_onetags + self.onetags += map( string.lower, self.onetags ) + self.twotags = valid_twotags + self.twotags += map( string.lower, self.twotags ) + self.deptags = deprecated_onetags + deprecated_twotags + self.deptags += map( string.lower, self.deptags ) + self.mode = 'strict_html' + elif mode == 'loose_html': + self.onetags = valid_onetags + deprecated_onetags + self.onetags += map( string.lower, self.onetags ) + self.twotags = valid_twotags + deprecated_twotags + self.twotags += map( string.lower, self.twotags ) + self.mode = mode + elif mode == 'xml': if onetags and twotags: self.onetags = onetags self.twotags = twotags @@ -178,8 +178,8 @@ class page: self.onetags = russell( ) self.twotags = russell( ) self.mode = mode - else: - raise ModeError( mode ) + else: + raise ModeError( mode ) def __getattr__( self, attr ): if attr.startswith("__") and attr.endswith("__"): @@ -192,7 +192,7 @@ class page: end = [ '</body>', '</html>' ] else: end = [ ] - + return self.separator.join( self.header + self.content + self.footer + end ) def __call__( self, escape=False ): @@ -444,37 +444,37 @@ class russell: """A dummy class that contains anything.""" def __contains__( self, item ): - return True + return True class MarkupError( Exception ): """All our exceptions subclass this.""" def __str__( self ): - return self.message + return self.message class ClosingError( MarkupError ): def __init__( self, tag ): - self.message = "The element '%s' does not accept non-keyword arguments (has no closing tag)." % tag + self.message = "The element '%s' does not accept non-keyword arguments (has no closing tag)." % tag class OpeningError( MarkupError ): def __init__( self, tag ): - self.message = "The element '%s' can not be opened." % tag + self.message = "The element '%s' can not be opened." % tag class ArgumentError( MarkupError ): def __init__( self, tag ): - self.message = "The element '%s' was called with more than one non-keyword argument." % tag + self.message = "The element '%s' was called with more than one non-keyword argument." % tag class InvalidElementError( MarkupError ): def __init__( self, tag, mode ): - self.message = "The element '%s' is not valid for your mode '%s'." % ( tag, mode ) + self.message = "The element '%s' is not valid for your mode '%s'." % ( tag, mode ) class DeprecationError( MarkupError ): def __init__( self, tag ): - self.message = "The element '%s' is deprecated, instantiate markup.page with mode='loose_html' to allow it." % tag + self.message = "The element '%s' is deprecated, instantiate markup.page with mode='loose_html' to allow it." % tag class ModeError( MarkupError ): def __init__( self, mode ): - self.message = "Mode '%s' is invalid, possible values: strict_html, loose_html, xml." % mode + self.message = "Mode '%s' is invalid, possible values: strict_html, loose_html, xml." % mode class CustomizationError( MarkupError ): def __init__( self ): diff --git a/tablib/packages/simplejson/__init__.py b/tablib/packages/simplejson/__init__.py deleted file mode 100644 index dcfd541..0000000 --- a/tablib/packages/simplejson/__init__.py +++ /dev/null @@ -1,437 +0,0 @@ -r"""JSON (JavaScript Object Notation) <http://json.org> is a subset of -JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data -interchange format. - -:mod:`simplejson` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained -version of the :mod:`json` library contained in Python 2.6, but maintains -compatibility with Python 2.4 and Python 2.5 and (currently) has -significant performance advantages, even without using the optional C -extension for speedups. - -Encoding basic Python object hierarchies:: - - >>> import simplejson as json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print json.dumps("\"foo\bar") - "\"foo\bar" - >>> print json.dumps(u'\u1234') - "\u1234" - >>> print json.dumps('\\') - "\\" - >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) - {"a": 0, "b": 0, "c": 0} - >>> from StringIO import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import simplejson as json - >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import simplejson as json - >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') - >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import simplejson as json - >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj - True - >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' - True - >>> from StringIO import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io)[0] == 'streaming API' - True - -Specializing JSON object decoding:: - - >>> import simplejson as json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> from decimal import Decimal - >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') - True - -Specializing JSON object encoding:: - - >>> import simplejson as json - >>> def encode_complex(obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... raise TypeError(repr(o) + " is not JSON serializable") - ... - >>> json.dumps(2 + 1j, default=encode_complex) - '[2.0, 1.0]' - >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) - '[2.0, 1.0]' - >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) - '[2.0, 1.0]' - - -Using simplejson.tool from the shell to validate and pretty-print:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) -""" -__version__ = '2.1.1' -__all__ = [ - 'dump', 'dumps', 'load', 'loads', - 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', - 'OrderedDict', -] - -__author__ = 'Bob Ippolito <bob@redivi.com>' - -from decimal import Decimal - -from decoder import JSONDecoder, JSONDecodeError -from encoder import JSONEncoder -def _import_OrderedDict(): - import collections - try: - return collections.OrderedDict - except AttributeError: - import ordered_dict - return ordered_dict.OrderedDict -OrderedDict = _import_OrderedDict() - -def _import_c_make_encoder(): - try: - from simplejson._speedups import make_encoder - return make_encoder - except ImportError: - return None - -_default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - use_decimal=False, -) - -def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=False, **kw): - """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a - ``.write()``-supporting file-like object). - - If ``skipkeys`` is true then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the some chunks written to ``fp`` - may be ``unicode`` instances, subject to normal Python ``str`` to - ``unicode`` coercion rules. Unless ``fp.write()`` explicitly - understands ``unicode`` (as in ``codecs.getwriter()``) this is likely - to cause an error. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) - in strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If *indent* is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``False``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not kw): - iterable = _default_encoder.iterencode(obj) - else: - if cls is None: - cls = JSONEncoder - iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, - default=default, use_decimal=use_decimal, **kw).iterencode(obj) - # could accelerate with writelines in some versions of Python, at - # a debuggability cost - for chunk in iterable: - fp.write(chunk) - - -def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, - allow_nan=True, cls=None, indent=None, separators=None, - encoding='utf-8', default=None, use_decimal=False, **kw): - """Serialize ``obj`` to a JSON formatted ``str``. - - If ``skipkeys`` is false then ``dict`` keys that are not basic types - (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) - will be skipped instead of raising a ``TypeError``. - - If ``ensure_ascii`` is false, then the return value will be a - ``unicode`` instance subject to normal Python ``str`` to ``unicode`` - coercion rules instead of being escaped to an ASCII ``str``. - - If ``check_circular`` is false, then the circular reference check - for container types will be skipped and a circular reference will - result in an ``OverflowError`` (or worse). - - If ``allow_nan`` is false, then it will be a ``ValueError`` to - serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in - strict compliance of the JSON specification, instead of using the - JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). - - If ``indent`` is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If ``separators`` is an ``(item_separator, dict_separator)`` tuple - then it will be used instead of the default ``(', ', ': ')`` separators. - ``(',', ':')`` is the most compact JSON representation. - - ``encoding`` is the character encoding for str instances, default is UTF-8. - - ``default(obj)`` is a function that should return a serializable version - of obj or raise TypeError. The default simply raises TypeError. - - If *use_decimal* is true (default: ``False``) then decimal.Decimal - will be natively serialized to JSON with full precision. - - To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the - ``.default()`` method to serialize additional types), specify it with - the ``cls`` kwarg. - - """ - # cached encoder - if (not skipkeys and ensure_ascii and - check_circular and allow_nan and - cls is None and indent is None and separators is None and - encoding == 'utf-8' and default is None and not use_decimal - and not kw): - return _default_encoder.encode(obj) - if cls is None: - cls = JSONEncoder - return cls( - skipkeys=skipkeys, ensure_ascii=ensure_ascii, - check_circular=check_circular, allow_nan=allow_nan, indent=indent, - separators=separators, encoding=encoding, default=default, - use_decimal=use_decimal, **kw).encode(obj) - - -_default_decoder = JSONDecoder(encoding=None, object_hook=None, - object_pairs_hook=None) - - -def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing - a JSON document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - return loads(fp.read(), - encoding=encoding, cls=cls, object_hook=object_hook, - parse_float=parse_float, parse_int=parse_int, - parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - use_decimal=use_decimal, **kw) - - -def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, object_pairs_hook=None, - use_decimal=False, **kw): - """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON - document) to a Python object. - - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - If *use_decimal* is true (default: ``False``) then it implies - parse_float=decimal.Decimal for parity with ``dump``. - - To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` - kwarg. - - """ - if (cls is None and encoding is None and object_hook is None and - parse_int is None and parse_float is None and - parse_constant is None and object_pairs_hook is None - and not use_decimal and not kw): - return _default_decoder.decode(s) - if cls is None: - cls = JSONDecoder - if object_hook is not None: - kw['object_hook'] = object_hook - if object_pairs_hook is not None: - kw['object_pairs_hook'] = object_pairs_hook - if parse_float is not None: - kw['parse_float'] = parse_float - if parse_int is not None: - kw['parse_int'] = parse_int - if parse_constant is not None: - kw['parse_constant'] = parse_constant - if use_decimal: - if parse_float is not None: - raise TypeError("use_decimal=True implies parse_float=Decimal") - kw['parse_float'] = Decimal - return cls(encoding=encoding, **kw).decode(s) - - -def _toggle_speedups(enabled): - import simplejson.decoder as dec - import simplejson.encoder as enc - import simplejson.scanner as scan - c_make_encoder = _import_c_make_encoder() - if enabled: - dec.scanstring = dec.c_scanstring or dec.py_scanstring - enc.c_make_encoder = c_make_encoder - enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or - enc.py_encode_basestring_ascii) - scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner - else: - dec.scanstring = dec.py_scanstring - enc.c_make_encoder = None - enc.encode_basestring_ascii = enc.py_encode_basestring_ascii - scan.make_scanner = scan.py_make_scanner - dec.make_scanner = scan.make_scanner - global _default_decoder - _default_decoder = JSONDecoder( - encoding=None, - object_hook=None, - object_pairs_hook=None, - ) - global _default_encoder - _default_encoder = JSONEncoder( - skipkeys=False, - ensure_ascii=True, - check_circular=True, - allow_nan=True, - indent=None, - separators=None, - encoding='utf-8', - default=None, - ) diff --git a/tablib/packages/simplejson/_speedups.c b/tablib/packages/simplejson/_speedups.c deleted file mode 100644 index b06ba50..0000000 --- a/tablib/packages/simplejson/_speedups.c +++ /dev/null @@ -1,2561 +0,0 @@ -#include "Python.h" -#include "structmember.h" -#if PY_VERSION_HEX < 0x02070000 && !defined(PyOS_string_to_double) -#define PyOS_string_to_double json_PyOS_string_to_double -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); -static double -json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) { - double x; - assert(endptr == NULL); - assert(overflow_exception == NULL); - PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) - x = PyOS_ascii_atof(s); - PyFPE_END_PROTECT(x) - return x; -} -#endif -#if PY_VERSION_HEX < 0x02060000 && !defined(Py_TYPE) -#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) -#endif -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#define PyInt_FromSsize_t PyInt_FromLong -#define PyInt_AsSsize_t PyInt_AsLong -#endif -#ifndef Py_IS_FINITE -#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) -#endif - -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - -#define DEFAULT_ENCODING "utf-8" - -#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) -#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) -#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) -#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) -#define Decimal_Check(op) (PyObject_TypeCheck(op, DecimalTypePtr)) - -static PyTypeObject PyScannerType; -static PyTypeObject PyEncoderType; -static PyTypeObject *DecimalTypePtr; - -typedef struct _PyScannerObject { - PyObject_HEAD - PyObject *encoding; - PyObject *strict; - PyObject *object_hook; - PyObject *pairs_hook; - PyObject *parse_float; - PyObject *parse_int; - PyObject *parse_constant; - PyObject *memo; -} PyScannerObject; - -static PyMemberDef scanner_members[] = { - {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, - {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"}, - {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, - {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, - {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, - {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, - {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, - {NULL} -}; - -typedef struct _PyEncoderObject { - PyObject_HEAD - PyObject *markers; - PyObject *defaultfn; - PyObject *encoder; - PyObject *indent; - PyObject *key_separator; - PyObject *item_separator; - PyObject *sort_keys; - PyObject *skipkeys; - PyObject *key_memo; - int fast_encode; - int allow_nan; - int use_decimal; -} PyEncoderObject; - -static PyMemberDef encoder_members[] = { - {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, - {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, - {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, - {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, - {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, - {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, - {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, - {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys), READONLY, "skipkeys"}, - {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, - {NULL} -}; - -static Py_ssize_t -ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars); -static PyObject * -ascii_escape_unicode(PyObject *pystr); -static PyObject * -ascii_escape_str(PyObject *pystr); -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); -void init_speedups(void); -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds); -static void -scanner_dealloc(PyObject *self); -static int -scanner_clear(PyObject *self); -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds); -static void -encoder_dealloc(PyObject *self); -static int -encoder_clear(PyObject *self); -static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level); -static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level); -static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level); -static PyObject * -_encoded_const(PyObject *obj); -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj); -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj); - -#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') -#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) - -#define MIN_EXPANSION 6 -#ifdef Py_UNICODE_WIDE -#define MAX_EXPANSION (2 * MIN_EXPANSION) -#else -#define MAX_EXPANSION MIN_EXPANSION -#endif - -static int -_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) -{ - /* PyObject to Py_ssize_t converter */ - *size_ptr = PyInt_AsSsize_t(o); - if (*size_ptr == -1 && PyErr_Occurred()) - return 0; - return 1; -} - -static PyObject * -_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) -{ - /* Py_ssize_t to PyObject converter */ - return PyInt_FromSsize_t(*size_ptr); -} - -static Py_ssize_t -ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) -{ - /* Escape unicode code point c to ASCII escape sequences - in char *output. output must have at least 12 bytes unused to - accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ - output[chars++] = '\\'; - switch (c) { - case '\\': output[chars++] = (char)c; break; - case '"': output[chars++] = (char)c; break; - case '\b': output[chars++] = 'b'; break; - case '\f': output[chars++] = 'f'; break; - case '\n': output[chars++] = 'n'; break; - case '\r': output[chars++] = 'r'; break; - case '\t': output[chars++] = 't'; break; - default: -#ifdef Py_UNICODE_WIDE - if (c >= 0x10000) { - /* UTF-16 surrogate pair */ - Py_UNICODE v = c - 0x10000; - c = 0xd800 | ((v >> 10) & 0x3ff); - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - c = 0xdc00 | (v & 0x3ff); - output[chars++] = '\\'; - } -#endif - output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; - } - return chars; -} - -static PyObject * -ascii_escape_unicode(PyObject *pystr) -{ - /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t max_output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - Py_UNICODE *input_unicode; - - input_chars = PyUnicode_GET_SIZE(pystr); - input_unicode = PyUnicode_AS_UNICODE(pystr); - - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; - max_output_size = 2 + (input_chars * MAX_EXPANSION); - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); - chars = 0; - output[chars++] = '"'; - for (i = 0; i < input_chars; i++) { - Py_UNICODE c = input_unicode[i]; - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - chars = ascii_escape_char(c, output, chars); - } - if (output_size - chars < (1 + MAX_EXPANSION)) { - /* There's more than four, so let's resize by a lot */ - Py_ssize_t new_output_size = output_size * 2; - /* This is an upper bound */ - if (new_output_size > max_output_size) { - new_output_size = max_output_size; - } - /* Make sure that the output size changed before resizing */ - if (new_output_size != output_size) { - output_size = new_output_size; - if (_PyString_Resize(&rval, output_size) == -1) { - return NULL; - } - output = PyString_AS_STRING(rval); - } - } - } - output[chars++] = '"'; - if (_PyString_Resize(&rval, chars) == -1) { - return NULL; - } - return rval; -} - -static PyObject * -ascii_escape_str(PyObject *pystr) -{ - /* Take a PyString pystr and return a new ASCII-only escaped PyString */ - Py_ssize_t i; - Py_ssize_t input_chars; - Py_ssize_t output_size; - Py_ssize_t chars; - PyObject *rval; - char *output; - char *input_str; - - input_chars = PyString_GET_SIZE(pystr); - input_str = PyString_AS_STRING(pystr); - - /* Fast path for a string that's already ASCII */ - for (i = 0; i < input_chars; i++) { - Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; - if (!S_CHAR(c)) { - /* If we have to escape something, scan the string for unicode */ - Py_ssize_t j; - for (j = i; j < input_chars; j++) { - c = (Py_UNICODE)(unsigned char)input_str[j]; - if (c > 0x7f) { - /* We hit a non-ASCII character, bail to unicode mode */ - PyObject *uni; - uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); - if (uni == NULL) { - return NULL; - } - rval = ascii_escape_unicode(uni); - Py_DECREF(uni); - return rval; - } - } - break; - } - } - - if (i == input_chars) { - /* Input is already ASCII */ - output_size = 2 + input_chars; - } - else { - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; - } - rval = PyString_FromStringAndSize(NULL, output_size); - if (rval == NULL) { - return NULL; - } - output = PyString_AS_STRING(rval); - output[0] = '"'; - - /* We know that everything up to i is ASCII already */ - chars = i + 1; - memcpy(&output[1], input_str, i); - - for (; i < input_chars; i++) { - Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; - if (S_CHAR(c)) { - output[chars++] = (char)c; - } - else { - chars = ascii_escape_char(c, output, chars); - } - /* An ASCII char can't possibly expand to a surrogate! */ - if (output_size - chars < (1 + MIN_EXPANSION)) { - /* There's more than four, so let's resize by a lot */ - output_size *= 2; - if (output_size > 2 + (input_chars * MIN_EXPANSION)) { - output_size = 2 + (input_chars * MIN_EXPANSION); - } - if (_PyString_Resize(&rval, output_size) == -1) { - return NULL; - } - output = PyString_AS_STRING(rval); - } - } - output[chars++] = '"'; - if (_PyString_Resize(&rval, chars) == -1) { - return NULL; - } - return rval; -} - -static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) -{ - /* Use the Python function simplejson.decoder.errmsg to raise a nice - looking ValueError exception */ - static PyObject *JSONDecodeError = NULL; - PyObject *exc; - if (JSONDecodeError == NULL) { - PyObject *decoder = PyImport_ImportModule("simplejson.decoder"); - if (decoder == NULL) - return; - JSONDecodeError = PyObject_GetAttrString(decoder, "JSONDecodeError"); - Py_DECREF(decoder); - if (JSONDecodeError == NULL) - return; - } - exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); - if (exc) { - PyErr_SetObject(JSONDecodeError, exc); - Py_DECREF(exc); - } -} - -static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - PyObject *ustr = PyUnicode_FromUnicode(NULL, 0); - if (ustr == NULL) - return NULL; - - joinfn = PyObject_GetAttrString(ustr, "join"); - Py_DECREF(ustr); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); -} - -static PyObject * -join_list_string(PyObject *lst) -{ - /* return ''.join(lst) */ - static PyObject *joinfn = NULL; - if (joinfn == NULL) { - PyObject *ustr = PyString_FromStringAndSize(NULL, 0); - if (ustr == NULL) - return NULL; - - joinfn = PyObject_GetAttrString(ustr, "join"); - Py_DECREF(ustr); - if (joinfn == NULL) - return NULL; - } - return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); -} - -static PyObject * -_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { - /* return (rval, idx) tuple, stealing reference to rval */ - PyObject *tpl; - PyObject *pyidx; - /* - steal a reference to rval, returns (rval, idx) - */ - if (rval == NULL) { - return NULL; - } - pyidx = PyInt_FromSsize_t(idx); - if (pyidx == NULL) { - Py_DECREF(rval); - return NULL; - } - tpl = PyTuple_New(2); - if (tpl == NULL) { - Py_DECREF(pyidx); - Py_DECREF(rval); - return NULL; - } - PyTuple_SET_ITEM(tpl, 0, rval); - PyTuple_SET_ITEM(tpl, 1, pyidx); - return tpl; -} - -#define APPEND_OLD_CHUNK \ - if (chunk != NULL) { \ - if (chunks == NULL) { \ - chunks = PyList_New(0); \ - if (chunks == NULL) { \ - goto bail; \ - } \ - } \ - if (PyList_Append(chunks, chunk)) { \ - goto bail; \ - } \ - Py_CLEAR(chunk); \ - } - -static PyObject * -scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyString pystr. - end is the index of the first character after the quote. - encoding is the encoding of pystr (must be an ASCII superset) - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyString (if ASCII-only) or PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyString_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - int has_unicode = 0; - char *buf = PyString_AS_STRING(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (end < 0 || len <= end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = (unsigned char)buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg("Invalid control character at", pystr, next); - goto bail; - } - else if (c > 0x7f) { - has_unicode = 1; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - PyObject *strchunk; - APPEND_OLD_CHUNK - strchunk = PyString_FromStringAndSize(&buf[end], next - end); - if (strchunk == NULL) { - goto bail; - } - if (has_unicode) { - chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); - Py_DECREF(strchunk); - if (chunk == NULL) { - goto bail; - } - } - else { - chunk = strchunk; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg("Invalid \\escape", pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - Py_UNICODE digit = buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } -#ifdef Py_UNICODE_WIDE - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - Py_UNICODE c2 = 0; - if (end + 6 >= len) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - if (buf[next++] != '\\' || buf[next++] != 'u') { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - end += 6; - /* Decode 4 hex digits */ - for (; next < end; next++) { - c2 <<= 4; - Py_UNICODE digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - else if ((c & 0xfc00) == 0xdc00) { - raise_errmsg("Unpaired low surrogate", pystr, end - 5); - goto bail; - } -#endif - } - if (c > 0x7f) { - has_unicode = 1; - } - APPEND_OLD_CHUNK - if (has_unicode) { - chunk = PyUnicode_FromUnicode(&c, 1); - if (chunk == NULL) { - goto bail; - } - } - else { - char c_char = Py_CHARMASK(c); - chunk = PyString_FromStringAndSize(&c_char, 1); - if (chunk == NULL) { - goto bail; - } - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = PyString_FromStringAndSize("", 0); - } - else { - APPEND_OLD_CHUNK - rval = join_list_string(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - - -static PyObject * -scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) -{ - /* Read the JSON string from PyUnicode pystr. - end is the index of the first character after the quote. - if strict is zero then literal control characters are allowed - *next_end_ptr is a return-by-reference index of the character - after the end quote - - Return value is a new PyUnicode - */ - PyObject *rval; - Py_ssize_t len = PyUnicode_GET_SIZE(pystr); - Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); - PyObject *chunks = NULL; - PyObject *chunk = NULL; - - if (end < 0 || len <= end) { - PyErr_SetString(PyExc_ValueError, "end is out of bounds"); - goto bail; - } - while (1) { - /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; - for (next = end; next < len; next++) { - c = buf[next]; - if (c == '"' || c == '\\') { - break; - } - else if (strict && c <= 0x1f) { - raise_errmsg("Invalid control character at", pystr, next); - goto bail; - } - } - if (!(c == '"' || c == '\\')) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - /* Pick up this chunk if it's not zero length */ - if (next != end) { - APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&buf[end], next - end); - if (chunk == NULL) { - goto bail; - } - } - next++; - if (c == '"') { - end = next; - break; - } - if (next == len) { - raise_errmsg("Unterminated string starting at", pystr, begin); - goto bail; - } - c = buf[next]; - if (c != 'u') { - /* Non-unicode backslash escapes */ - end = next + 1; - switch (c) { - case '"': break; - case '\\': break; - case '/': break; - case 'b': c = '\b'; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - default: c = 0; - } - if (c == 0) { - raise_errmsg("Invalid \\escape", pystr, end - 2); - goto bail; - } - } - else { - c = 0; - next++; - end = next + 4; - if (end >= len) { - raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); - goto bail; - } - /* Decode 4 hex digits */ - for (; next < end; next++) { - Py_UNICODE digit = buf[next]; - c <<= 4; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } -#ifdef Py_UNICODE_WIDE - /* Surrogate pair */ - if ((c & 0xfc00) == 0xd800) { - Py_UNICODE c2 = 0; - if (end + 6 >= len) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - if (buf[next++] != '\\' || buf[next++] != 'u') { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - end += 6; - /* Decode 4 hex digits */ - for (; next < end; next++) { - c2 <<= 4; - Py_UNICODE digit = buf[next]; - switch (digit) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - c2 |= (digit - '0'); break; - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': - c2 |= (digit - 'a' + 10); break; - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': - c2 |= (digit - 'A' + 10); break; - default: - raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); - goto bail; - } - } - if ((c2 & 0xfc00) != 0xdc00) { - raise_errmsg("Unpaired high surrogate", pystr, end - 5); - goto bail; - } - c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); - } - else if ((c & 0xfc00) == 0xdc00) { - raise_errmsg("Unpaired low surrogate", pystr, end - 5); - goto bail; - } -#endif - } - APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&c, 1); - if (chunk == NULL) { - goto bail; - } - } - - if (chunks == NULL) { - if (chunk != NULL) - rval = chunk; - else - rval = PyUnicode_FromUnicode(NULL, 0); - } - else { - APPEND_OLD_CHUNK - rval = join_list_unicode(chunks); - if (rval == NULL) { - goto bail; - } - Py_CLEAR(chunks); - } - *next_end_ptr = end; - return rval; -bail: - *next_end_ptr = -1; - Py_XDECREF(chunk); - Py_XDECREF(chunks); - return NULL; -} - -PyDoc_STRVAR(pydoc_scanstring, - "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" - "\n" - "Scan the string s for a JSON string. End is the index of the\n" - "character in s after the quote that started the JSON string.\n" - "Unescapes all valid JSON string escape sequences and raises ValueError\n" - "on attempt to decode an invalid string. If strict is False then literal\n" - "control characters are allowed in the string.\n" - "\n" - "Returns a tuple of the decoded string and the index of the character in s\n" - "after the end quote." -); - -static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) -{ - PyObject *pystr; - PyObject *rval; - Py_ssize_t end; - Py_ssize_t next_end = -1; - char *encoding = NULL; - int strict = 1; - if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { - return NULL; - } - if (encoding == NULL) { - encoding = DEFAULT_ENCODING; - } - if (PyString_Check(pystr)) { - rval = scanstring_str(pystr, end, encoding, strict, &next_end); - } - else if (PyUnicode_Check(pystr)) { - rval = scanstring_unicode(pystr, end, strict, &next_end); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - return _build_rval_index_tuple(rval, next_end); -} - -PyDoc_STRVAR(pydoc_encode_basestring_ascii, - "encode_basestring_ascii(basestring) -> str\n" - "\n" - "Return an ASCII-only JSON representation of a Python string" -); - -static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) -{ - /* Return an ASCII-only JSON representation of a Python string */ - /* METH_O */ - if (PyString_Check(pystr)) { - return ascii_escape_str(pystr); - } - else if (PyUnicode_Check(pystr)) { - return ascii_escape_unicode(pystr); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } -} - -static void -scanner_dealloc(PyObject *self) -{ - /* Deallocate scanner object */ - scanner_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -scanner_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_VISIT(s->encoding); - Py_VISIT(s->strict); - Py_VISIT(s->object_hook); - Py_VISIT(s->pairs_hook); - Py_VISIT(s->parse_float); - Py_VISIT(s->parse_int); - Py_VISIT(s->parse_constant); - Py_VISIT(s->memo); - return 0; -} - -static int -scanner_clear(PyObject *self) -{ - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - Py_CLEAR(s->memo); - return 0; -} - -static PyObject * -_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON object from PyString pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook or - object_pairs_hook can change that) - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - char *encoding = PyString_AS_STRING(s->encoding); - int strict = PyObject_IsTrue(s->strict); - int has_pairs_hook = (s->pairs_hook != Py_None); - Py_ssize_t next_idx; - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - while (idx <= end_idx) { - PyObject *memokey; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg("Expecting property name", pystr, idx); - goto bail; - } - key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg("Expecting : delimiter", pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON data type */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -static PyObject * -_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON object from PyUnicode pystr. - idx is the index of the first character after the opening curly brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing curly brace. - - Returns a new PyObject (usually a dict, but object_hook can change that) - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; - PyObject *rval = NULL; - PyObject *pairs = NULL; - PyObject *item; - PyObject *key = NULL; - PyObject *val = NULL; - int strict = PyObject_IsTrue(s->strict); - int has_pairs_hook = (s->pairs_hook != Py_None); - Py_ssize_t next_idx; - - if (has_pairs_hook) { - pairs = PyList_New(0); - if (pairs == NULL) - return NULL; - } - else { - rval = PyDict_New(); - if (rval == NULL) - return NULL; - } - - /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { - while (idx <= end_idx) { - PyObject *memokey; - - /* read key */ - if (str[idx] != '"') { - raise_errmsg("Expecting property name", pystr, idx); - goto bail; - } - key = scanstring_unicode(pystr, idx + 1, strict, &next_idx); - if (key == NULL) - goto bail; - memokey = PyDict_GetItem(s->memo, key); - if (memokey != NULL) { - Py_INCREF(memokey); - Py_DECREF(key); - key = memokey; - } - else { - if (PyDict_SetItem(s->memo, key, key) < 0) - goto bail; - } - idx = next_idx; - - /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { - raise_errmsg("Expecting : delimiter", pystr, idx); - goto bail; - } - idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) - goto bail; - - if (has_pairs_hook) { - item = PyTuple_Pack(2, key, val); - if (item == NULL) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - if (PyList_Append(pairs, item) == -1) { - Py_DECREF(item); - goto bail; - } - Py_DECREF(item); - } - else { - if (PyDict_SetItem(rval, key, val) < 0) - goto bail; - Py_CLEAR(key); - Py_CLEAR(val); - } - idx = next_idx; - - /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the object is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == '}') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - - /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - - /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ - if (s->pairs_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); - if (val == NULL) - goto bail; - Py_DECREF(pairs); - *next_idx_ptr = idx + 1; - return val; - } - - /* if object_hook is not None: rval = object_hook(rval) */ - if (s->object_hook != Py_None) { - val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); - if (val == NULL) - goto bail; - Py_DECREF(rval); - rval = val; - val = NULL; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(rval); - Py_XDECREF(key); - Py_XDECREF(val); - Py_XDECREF(pairs); - return NULL; -} - -static PyObject * -_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - while (idx <= end_idx) { - - /* read any JSON term and de-tuplefy the (rval, idx) */ - val = scan_once_str(s, pystr, idx, &next_idx); - if (val == NULL) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - raise_errmsg("Expecting object", pystr, idx); - } - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON array from PyString pystr. - idx is the index of the first character after the opening brace. - *next_idx_ptr is a return-by-reference index to the first character after - the closing brace. - - Returns a new PyList - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; - PyObject *val = NULL; - PyObject *rval = PyList_New(0); - Py_ssize_t next_idx; - if (rval == NULL) - return NULL; - - /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { - while (idx <= end_idx) { - - /* read any JSON term */ - val = scan_once_unicode(s, pystr, idx, &next_idx); - if (val == NULL) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - raise_errmsg("Expecting object", pystr, idx); - } - goto bail; - } - - if (PyList_Append(rval, val) == -1) - goto bail; - - Py_CLEAR(val); - idx = next_idx; - - /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - - /* bail if the array is closed or we didn't get the , delimiter */ - if (idx > end_idx) break; - if (str[idx] == ']') { - break; - } - else if (str[idx] != ',') { - raise_errmsg("Expecting , delimiter", pystr, idx); - goto bail; - } - idx++; - - /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - } - } - - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { - raise_errmsg("Expecting object", pystr, end_idx); - goto bail; - } - *next_idx_ptr = idx + 1; - return rval; -bail: - Py_XDECREF(val); - Py_DECREF(rval); - return NULL; -} - -static PyObject * -_parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { - /* Read a JSON constant from PyString pystr. - constant is the constant string that was found - ("NaN", "Infinity", "-Infinity"). - idx is the index of the first character of the constant - *next_idx_ptr is a return-by-reference index to the first character after - the constant. - - Returns the result of parse_constant - */ - PyObject *cstr; - PyObject *rval; - /* constant is "NaN", "Infinity", or "-Infinity" */ - cstr = PyString_InternFromString(constant); - if (cstr == NULL) - return NULL; - - /* rval = parse_constant(constant) */ - rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL); - idx += PyString_GET_SIZE(cstr); - Py_DECREF(cstr); - *next_idx_ptr = idx; - return rval; -} - -static PyObject * -_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { - /* Read a JSON number from PyString pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - idx++; - if (idx > end_idx) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - - /* save the index of the 'e' or 'E' just in case we need to backtrack */ - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyString_FromStringAndSize(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ - double d = PyOS_string_to_double(PyString_AS_STRING(numstr), - NULL, NULL); - if (d == -1.0 && PyErr_Occurred()) - return NULL; - rval = PyFloat_FromDouble(d); - } - } - else { - /* parse as an int using a fast path if available, otherwise call user defined method */ - if (s->parse_int != (PyObject *)&PyInt_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); - } - else { - rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); - } - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -static PyObject * -_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { - /* Read a JSON number from PyUnicode pystr. - idx is the index of the first character of the number - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of that number: - PyInt, PyLong, or PyFloat. - May return other types if parse_int or parse_float are set - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; - Py_ssize_t idx = start; - int is_float = 0; - PyObject *rval; - PyObject *numstr; - - /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { - idx++; - if (idx > end_idx) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - } - - /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { - idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { - idx++; - } - /* no integer digits, error */ - else { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - - /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { - is_float = 1; - idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - } - - /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { - Py_ssize_t e_start = idx; - idx++; - - /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; - - /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; - - /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { - is_float = 1; - } - else { - idx = e_start; - } - } - - /* copy the section we determined to be a number */ - numstr = PyUnicode_FromUnicode(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - rval = PyFloat_FromString(numstr, NULL); - } - } - else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); - } - Py_DECREF(numstr); - *next_idx_ptr = idx; - return rval; -} - -static PyObject * -scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyString pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - char *str = PyString_AS_STRING(pystr); - Py_ssize_t length = PyString_GET_SIZE(pystr); - if (idx >= length) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - return scanstring_str(pystr, idx + 1, - PyString_AS_STRING(s->encoding), - PyObject_IsTrue(s->strict), - next_idx_ptr); - case '{': - /* object */ - return _parse_object_str(s, pystr, idx + 1, next_idx_ptr); - case '[': - /* array */ - return _parse_array_str(s, pystr, idx + 1, next_idx_ptr); - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - return Py_None; - } - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - return Py_True; - } - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - return Py_False; - } - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - return _parse_constant(s, "NaN", idx, next_idx_ptr); - } - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - return _parse_constant(s, "Infinity", idx, next_idx_ptr); - } - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - return _parse_constant(s, "-Infinity", idx, next_idx_ptr); - } - break; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - return _match_number_str(s, pystr, idx, next_idx_ptr); -} - -static PyObject * -scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) -{ - /* Read one JSON term (of any kind) from PyUnicode pystr. - idx is the index of the first character of the term - *next_idx_ptr is a return-by-reference index to the first character after - the number. - - Returns a new PyObject representation of the term. - */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t length = PyUnicode_GET_SIZE(pystr); - if (idx >= length) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; - } - switch (str[idx]) { - case '"': - /* string */ - return scanstring_unicode(pystr, idx + 1, - PyObject_IsTrue(s->strict), - next_idx_ptr); - case '{': - /* object */ - return _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); - case '[': - /* array */ - return _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); - case 'n': - /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { - Py_INCREF(Py_None); - *next_idx_ptr = idx + 4; - return Py_None; - } - break; - case 't': - /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { - Py_INCREF(Py_True); - *next_idx_ptr = idx + 4; - return Py_True; - } - break; - case 'f': - /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { - Py_INCREF(Py_False); - *next_idx_ptr = idx + 5; - return Py_False; - } - break; - case 'N': - /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { - return _parse_constant(s, "NaN", idx, next_idx_ptr); - } - break; - case 'I': - /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { - return _parse_constant(s, "Infinity", idx, next_idx_ptr); - } - break; - case '-': - /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { - return _parse_constant(s, "-Infinity", idx, next_idx_ptr); - } - break; - } - /* Didn't find a string, object, array, or named constant. Look for a number. */ - return _match_number_unicode(s, pystr, idx, next_idx_ptr); -} - -static PyObject * -scanner_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to scan_once_{str,unicode} */ - PyObject *pystr; - PyObject *rval; - Py_ssize_t idx; - Py_ssize_t next_idx = -1; - static char *kwlist[] = {"string", "idx", NULL}; - PyScannerObject *s; - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) - return NULL; - - if (PyString_Check(pystr)) { - rval = scan_once_str(s, pystr, idx, &next_idx); - } - else if (PyUnicode_Check(pystr)) { - rval = scan_once_unicode(s, pystr, idx, &next_idx); - } - else { - PyErr_Format(PyExc_TypeError, - "first argument must be a string, not %.80s", - Py_TYPE(pystr)->tp_name); - return NULL; - } - PyDict_Clear(s->memo); - return _build_rval_index_tuple(rval, next_idx); -} - -static PyObject * -scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyScannerObject *s; - s = (PyScannerObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->encoding = NULL; - s->strict = NULL; - s->object_hook = NULL; - s->pairs_hook = NULL; - s->parse_float = NULL; - s->parse_int = NULL; - s->parse_constant = NULL; - } - return (PyObject *)s; -} - -static int -scanner_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Initialize Scanner object */ - PyObject *ctx; - static char *kwlist[] = {"context", NULL}; - PyScannerObject *s; - - assert(PyScanner_Check(self)); - s = (PyScannerObject *)self; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) - return -1; - - if (s->memo == NULL) { - s->memo = PyDict_New(); - if (s->memo == NULL) - goto bail; - } - - /* PyString_AS_STRING is used on encoding */ - s->encoding = PyObject_GetAttrString(ctx, "encoding"); - if (s->encoding == NULL) - goto bail; - if (s->encoding == Py_None) { - Py_DECREF(Py_None); - s->encoding = PyString_InternFromString(DEFAULT_ENCODING); - } - else if (PyUnicode_Check(s->encoding)) { - PyObject *tmp = PyUnicode_AsEncodedString(s->encoding, NULL, NULL); - Py_DECREF(s->encoding); - s->encoding = tmp; - } - if (s->encoding == NULL || !PyString_Check(s->encoding)) - goto bail; - - /* All of these will fail "gracefully" so we don't need to verify them */ - s->strict = PyObject_GetAttrString(ctx, "strict"); - if (s->strict == NULL) - goto bail; - s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); - if (s->object_hook == NULL) - goto bail; - s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); - if (s->pairs_hook == NULL) - goto bail; - s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); - if (s->parse_float == NULL) - goto bail; - s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); - if (s->parse_int == NULL) - goto bail; - s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); - if (s->parse_constant == NULL) - goto bail; - - return 0; - -bail: - Py_CLEAR(s->encoding); - Py_CLEAR(s->strict); - Py_CLEAR(s->object_hook); - Py_CLEAR(s->pairs_hook); - Py_CLEAR(s->parse_float); - Py_CLEAR(s->parse_int); - Py_CLEAR(s->parse_constant); - return -1; -} - -PyDoc_STRVAR(scanner_doc, "JSON scanner object"); - -static -PyTypeObject PyScannerType = { - PyObject_HEAD_INIT(NULL) - 0, /* tp_internal */ - "simplejson._speedups.Scanner", /* tp_name */ - sizeof(PyScannerObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - scanner_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - scanner_call, /* tp_call */ - 0, /* tp_str */ - 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ - 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - scanner_doc, /* tp_doc */ - scanner_traverse, /* tp_traverse */ - scanner_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - scanner_init, /* tp_init */ - 0,/* PyType_GenericAlloc, */ /* tp_alloc */ - scanner_new, /* tp_new */ - 0,/* PyObject_GC_Del, */ /* tp_free */ -}; - -static PyObject * -encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyEncoderObject *s; - s = (PyEncoderObject *)type->tp_alloc(type, 0); - if (s != NULL) { - s->markers = NULL; - s->defaultfn = NULL; - s->encoder = NULL; - s->indent = NULL; - s->key_separator = NULL; - s->item_separator = NULL; - s->sort_keys = NULL; - s->skipkeys = NULL; - s->key_memo = NULL; - } - return (PyObject *)s; -} - -static int -encoder_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* initialize Encoder object */ - static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", "key_memo", "use_decimal", NULL}; - - PyEncoderObject *s; - PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; - PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo, *use_decimal; - - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOO:make_encoder", kwlist, - &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, - &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal)) - return -1; - - s->markers = markers; - s->defaultfn = defaultfn; - s->encoder = encoder; - s->indent = indent; - s->key_separator = key_separator; - s->item_separator = item_separator; - s->sort_keys = sort_keys; - s->skipkeys = skipkeys; - s->key_memo = key_memo; - s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); - s->allow_nan = PyObject_IsTrue(allow_nan); - s->use_decimal = PyObject_IsTrue(use_decimal); - - Py_INCREF(s->markers); - Py_INCREF(s->defaultfn); - Py_INCREF(s->encoder); - Py_INCREF(s->indent); - Py_INCREF(s->key_separator); - Py_INCREF(s->item_separator); - Py_INCREF(s->sort_keys); - Py_INCREF(s->skipkeys); - Py_INCREF(s->key_memo); - return 0; -} - -static PyObject * -encoder_call(PyObject *self, PyObject *args, PyObject *kwds) -{ - /* Python callable interface to encode_listencode_obj */ - static char *kwlist[] = {"obj", "_current_indent_level", NULL}; - PyObject *obj; - PyObject *rval; - Py_ssize_t indent_level; - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, - &obj, _convertPyInt_AsSsize_t, &indent_level)) - return NULL; - rval = PyList_New(0); - if (rval == NULL) - return NULL; - if (encoder_listencode_obj(s, rval, obj, indent_level)) { - Py_DECREF(rval); - return NULL; - } - return rval; -} - -static PyObject * -_encoded_const(PyObject *obj) -{ - /* Return the JSON string representation of None, True, False */ - if (obj == Py_None) { - static PyObject *s_null = NULL; - if (s_null == NULL) { - s_null = PyString_InternFromString("null"); - } - Py_INCREF(s_null); - return s_null; - } - else if (obj == Py_True) { - static PyObject *s_true = NULL; - if (s_true == NULL) { - s_true = PyString_InternFromString("true"); - } - Py_INCREF(s_true); - return s_true; - } - else if (obj == Py_False) { - static PyObject *s_false = NULL; - if (s_false == NULL) { - s_false = PyString_InternFromString("false"); - } - Py_INCREF(s_false); - return s_false; - } - else { - PyErr_SetString(PyExc_ValueError, "not a const"); - return NULL; - } -} - -static PyObject * -encoder_encode_float(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a PyFloat */ - double i = PyFloat_AS_DOUBLE(obj); - if (!Py_IS_FINITE(i)) { - if (!s->allow_nan) { - PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); - return NULL; - } - if (i > 0) { - return PyString_FromString("Infinity"); - } - else if (i < 0) { - return PyString_FromString("-Infinity"); - } - else { - return PyString_FromString("NaN"); - } - } - /* Use a better float format here? */ - return PyObject_Repr(obj); -} - -static PyObject * -encoder_encode_string(PyEncoderObject *s, PyObject *obj) -{ - /* Return the JSON representation of a string */ - if (s->fast_encode) - return py_encode_basestring_ascii(NULL, obj); - else - return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); -} - -static int -_steal_list_append(PyObject *lst, PyObject *stolen) -{ - /* Append stolen and then decrement its reference count */ - int rval = PyList_Append(lst, stolen); - Py_DECREF(stolen); - return rval; -} - -static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level) -{ - /* Encode Python object obj to a JSON term, rval is a PyList */ - PyObject *newobj; - int rv; - - if (obj == Py_None || obj == Py_True || obj == Py_False) { - PyObject *cstr = _encoded_const(obj); - if (cstr == NULL) - return -1; - return _steal_list_append(rval, cstr); - } - else if (PyString_Check(obj) || PyUnicode_Check(obj)) - { - PyObject *encoded = encoder_encode_string(s, obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else if (PyInt_Check(obj) || PyLong_Check(obj)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else if (PyFloat_Check(obj)) { - PyObject *encoded = encoder_encode_float(s, obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else if (PyList_Check(obj) || PyTuple_Check(obj)) { - return encoder_listencode_list(s, rval, obj, indent_level); - } - else if (PyDict_Check(obj)) { - return encoder_listencode_dict(s, rval, obj, indent_level); - } - else if (s->use_decimal && Decimal_Check(obj)) { - PyObject *encoded = PyObject_Str(obj); - if (encoded == NULL) - return -1; - return _steal_list_append(rval, encoded); - } - else { - PyObject *ident = NULL; - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(obj); - if (ident == NULL) - return -1; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - Py_DECREF(ident); - return -1; - } - if (PyDict_SetItem(s->markers, ident, obj)) { - Py_DECREF(ident); - return -1; - } - } - newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL); - if (newobj == NULL) { - Py_XDECREF(ident); - return -1; - } - rv = encoder_listencode_obj(s, rval, newobj, indent_level); - Py_DECREF(newobj); - if (rv) { - Py_XDECREF(ident); - return -1; - } - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) { - Py_XDECREF(ident); - return -1; - } - Py_XDECREF(ident); - } - return rv; - } -} - -static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level) -{ - /* Encode Python dict dct a JSON term, rval is a PyList */ - static PyObject *open_dict = NULL; - static PyObject *close_dict = NULL; - static PyObject *empty_dict = NULL; - static PyObject *iteritems = NULL; - PyObject *kstr = NULL; - PyObject *ident = NULL; - PyObject *key, *value; - PyObject *iter = NULL; - PyObject *item = NULL; - PyObject *encoded = NULL; - int skipkeys; - Py_ssize_t idx; - - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL || iteritems == NULL) { - open_dict = PyString_InternFromString("{"); - close_dict = PyString_InternFromString("}"); - empty_dict = PyString_InternFromString("{}"); - iteritems = PyString_InternFromString("iteritems"); - if (open_dict == NULL || close_dict == NULL || empty_dict == NULL || iteritems == NULL) - return -1; - } - if (PyDict_Size(dct) == 0) - return PyList_Append(rval, empty_dict); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(dct); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, dct)) { - goto bail; - } - } - - if (PyList_Append(rval, open_dict)) - goto bail; - - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - - /* TODO: C speedup not implemented for sort_keys */ - - skipkeys = PyObject_IsTrue(s->skipkeys); - idx = 0; - iter = PyObject_CallMethodObjArgs(dct, iteritems, NULL); - if (iter == NULL) - goto bail; - while ((item = PyIter_Next(iter))) { - - key = PyTuple_GetItem(item, 0); - if (key == NULL) - goto bail; - value = PyTuple_GetItem(item, 1); - if (value == NULL) - goto bail; - - encoded = PyDict_GetItem(s->key_memo, key); - if (encoded != NULL) { - Py_INCREF(encoded); - } - else if (PyString_Check(key) || PyUnicode_Check(key)) { - Py_INCREF(key); - kstr = key; - } - else if (PyFloat_Check(key)) { - kstr = encoder_encode_float(s, key); - if (kstr == NULL) - goto bail; - } - else if (PyInt_Check(key) || PyLong_Check(key)) { - kstr = PyObject_Str(key); - if (kstr == NULL) - goto bail; - } - else if (key == Py_True || key == Py_False || key == Py_None) { - kstr = _encoded_const(key); - if (kstr == NULL) - goto bail; - } - else if (skipkeys) { - Py_DECREF(item); - continue; - } - else { - /* TODO: include repr of key */ - PyErr_SetString(PyExc_ValueError, "keys must be a string"); - goto bail; - } - - if (idx) { - if (PyList_Append(rval, s->item_separator)) - goto bail; - } - - if (encoded == NULL) { - encoded = encoder_encode_string(s, kstr); - Py_CLEAR(kstr); - if (encoded == NULL) - goto bail; - if (PyDict_SetItem(s->key_memo, key, encoded)) - goto bail; - } - if (PyList_Append(rval, encoded)) { - goto bail; - } - Py_CLEAR(encoded); - if (PyList_Append(rval, s->key_separator)) - goto bail; - if (encoder_listencode_obj(s, rval, value, indent_level)) - goto bail; - Py_CLEAR(item); - idx += 1; - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (PyList_Append(rval, close_dict)) - goto bail; - return 0; - -bail: - Py_XDECREF(encoded); - Py_XDECREF(item); - Py_XDECREF(iter); - Py_XDECREF(kstr); - Py_XDECREF(ident); - return -1; -} - - -static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level) -{ - /* Encode Python list seq to a JSON term, rval is a PyList */ - static PyObject *open_array = NULL; - static PyObject *close_array = NULL; - static PyObject *empty_array = NULL; - PyObject *ident = NULL; - PyObject *iter = NULL; - PyObject *obj = NULL; - int is_true; - int i = 0; - - if (open_array == NULL || close_array == NULL || empty_array == NULL) { - open_array = PyString_InternFromString("["); - close_array = PyString_InternFromString("]"); - empty_array = PyString_InternFromString("[]"); - if (open_array == NULL || close_array == NULL || empty_array == NULL) - return -1; - } - ident = NULL; - is_true = PyObject_IsTrue(seq); - if (is_true == -1) - return -1; - else if (is_true == 0) - return PyList_Append(rval, empty_array); - - if (s->markers != Py_None) { - int has_key; - ident = PyLong_FromVoidPtr(seq); - if (ident == NULL) - goto bail; - has_key = PyDict_Contains(s->markers, ident); - if (has_key) { - if (has_key != -1) - PyErr_SetString(PyExc_ValueError, "Circular reference detected"); - goto bail; - } - if (PyDict_SetItem(s->markers, ident, seq)) { - goto bail; - } - } - - iter = PyObject_GetIter(seq); - if (iter == NULL) - goto bail; - - if (PyList_Append(rval, open_array)) - goto bail; - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - */ - } - while ((obj = PyIter_Next(iter))) { - if (i) { - if (PyList_Append(rval, s->item_separator)) - goto bail; - } - if (encoder_listencode_obj(s, rval, obj, indent_level)) - goto bail; - i++; - Py_CLEAR(obj); - } - Py_CLEAR(iter); - if (PyErr_Occurred()) - goto bail; - if (ident != NULL) { - if (PyDict_DelItem(s->markers, ident)) - goto bail; - Py_CLEAR(ident); - } - if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level -= 1; - /* - yield '\n' + (_indent * _current_indent_level) - */ - } - if (PyList_Append(rval, close_array)) - goto bail; - return 0; - -bail: - Py_XDECREF(obj); - Py_XDECREF(iter); - Py_XDECREF(ident); - return -1; -} - -static void -encoder_dealloc(PyObject *self) -{ - /* Deallocate Encoder */ - encoder_clear(self); - Py_TYPE(self)->tp_free(self); -} - -static int -encoder_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_VISIT(s->markers); - Py_VISIT(s->defaultfn); - Py_VISIT(s->encoder); - Py_VISIT(s->indent); - Py_VISIT(s->key_separator); - Py_VISIT(s->item_separator); - Py_VISIT(s->sort_keys); - Py_VISIT(s->skipkeys); - Py_VISIT(s->key_memo); - return 0; -} - -static int -encoder_clear(PyObject *self) -{ - /* Deallocate Encoder */ - PyEncoderObject *s; - assert(PyEncoder_Check(self)); - s = (PyEncoderObject *)self; - Py_CLEAR(s->markers); - Py_CLEAR(s->defaultfn); - Py_CLEAR(s->encoder); - Py_CLEAR(s->indent); - Py_CLEAR(s->key_separator); - Py_CLEAR(s->item_separator); - Py_CLEAR(s->sort_keys); - Py_CLEAR(s->skipkeys); - Py_CLEAR(s->key_memo); - return 0; -} - -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); - -static -PyTypeObject PyEncoderType = { - PyObject_HEAD_INIT(NULL) - 0, /* tp_internal */ - "simplejson._speedups.Encoder", /* tp_name */ - sizeof(PyEncoderObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - encoder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - encoder_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - encoder_doc, /* tp_doc */ - encoder_traverse, /* tp_traverse */ - encoder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - encoder_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - encoder_init, /* tp_init */ - 0, /* tp_alloc */ - encoder_new, /* tp_new */ - 0, /* tp_free */ -}; - -static PyMethodDef speedups_methods[] = { - {"encode_basestring_ascii", - (PyCFunction)py_encode_basestring_ascii, - METH_O, - pydoc_encode_basestring_ascii}, - {"scanstring", - (PyCFunction)py_scanstring, - METH_VARARGS, - pydoc_scanstring}, - {NULL, NULL, 0, NULL} -}; - -PyDoc_STRVAR(module_doc, -"simplejson speedups\n"); - -void -init_speedups(void) -{ - PyObject *m, *decimal; - PyScannerType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyScannerType) < 0) - return; - PyEncoderType.tp_new = PyType_GenericNew; - if (PyType_Ready(&PyEncoderType) < 0) - return; - - decimal = PyImport_ImportModule("decimal"); - if (decimal == NULL) - return; - DecimalTypePtr = (PyTypeObject*)PyObject_GetAttrString(decimal, "Decimal"); - Py_DECREF(decimal); - if (DecimalTypePtr == NULL) - return; - - m = Py_InitModule3("_speedups", speedups_methods, module_doc); - Py_INCREF((PyObject*)&PyScannerType); - PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); - Py_INCREF((PyObject*)&PyEncoderType); - PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); -} diff --git a/tablib/packages/simplejson/decoder.py b/tablib/packages/simplejson/decoder.py deleted file mode 100644 index 4cf4015..0000000 --- a/tablib/packages/simplejson/decoder.py +++ /dev/null @@ -1,421 +0,0 @@ -"""Implementation of JSONDecoder -""" -import re -import sys -import struct - -from simplejson.scanner import make_scanner -def _import_c_scanstring(): - try: - from simplejson._speedups import scanstring - return scanstring - except ImportError: - return None -c_scanstring = _import_c_scanstring() - -__all__ = ['JSONDecoder'] - -FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL - -def _floatconstants(): - _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') - # The struct module in Python 2.4 would get frexp() out of range here - # when an endian is specified in the format string. Fixed in Python 2.5+ - if sys.byteorder != 'big': - _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] - nan, inf = struct.unpack('dd', _BYTES) - return nan, inf, -inf - -NaN, PosInf, NegInf = _floatconstants() - - -class JSONDecodeError(ValueError): - """Subclass of ValueError with the following additional properties: - - msg: The unformatted error message - doc: The JSON document being parsed - pos: The start index of doc where parsing failed - end: The end index of doc where parsing failed (may be None) - lineno: The line corresponding to pos - colno: The column corresponding to pos - endlineno: The line corresponding to end (may be None) - endcolno: The column corresponding to end (may be None) - - """ - def __init__(self, msg, doc, pos, end=None): - ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) - self.msg = msg - self.doc = doc - self.pos = pos - self.end = end - self.lineno, self.colno = linecol(doc, pos) - if end is not None: - self.endlineno, self.endcolno = linecol(doc, pos) - else: - self.endlineno, self.endcolno = None, None - - -def linecol(doc, pos): - lineno = doc.count('\n', 0, pos) + 1 - if lineno == 1: - colno = pos - else: - colno = pos - doc.rindex('\n', 0, pos) - return lineno, colno - - -def errmsg(msg, doc, pos, end=None): - # Note that this function is called from _speedups - lineno, colno = linecol(doc, pos) - if end is None: - #fmt = '{0}: line {1} column {2} (char {3})' - #return fmt.format(msg, lineno, colno, pos) - fmt = '%s: line %d column %d (char %d)' - return fmt % (msg, lineno, colno, pos) - endlineno, endcolno = linecol(doc, end) - #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' - #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) - fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' - return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) - - -_CONSTANTS = { - '-Infinity': NegInf, - 'Infinity': PosInf, - 'NaN': NaN, -} - -STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) -BACKSLASH = { - '"': u'"', '\\': u'\\', '/': u'/', - 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', -} - -DEFAULT_ENCODING = "utf-8" - -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match): - """Scan the string s for a JSON string. End is the index of the - character in s after the quote that started the JSON string. - Unescapes all valid JSON string escape sequences and raises ValueError - on attempt to decode an invalid string. If strict is False then literal - control characters are allowed in the string. - - Returns a tuple of the decoded string and the index of the character in s - after the end quote.""" - if encoding is None: - encoding = DEFAULT_ENCODING - chunks = [] - _append = chunks.append - begin = end - 1 - while 1: - chunk = _m(s, end) - if chunk is None: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - end = chunk.end() - content, terminator = chunk.groups() - # Content is contains zero or more unescaped string characters - if content: - if not isinstance(content, unicode): - content = unicode(content, encoding) - _append(content) - # Terminator is the end of string, a literal control character, - # or a backslash denoting that an escape sequence follows - if terminator == '"': - break - elif terminator != '\\': - if strict: - msg = "Invalid control character %r at" % (terminator,) - #msg = "Invalid control character {0!r} at".format(terminator) - raise JSONDecodeError(msg, s, end) - else: - _append(terminator) - continue - try: - esc = s[end] - except IndexError: - raise JSONDecodeError( - "Unterminated string starting at", s, begin) - # If not a unicode escape sequence, must be in the lookup table - if esc != 'u': - try: - char = _b[esc] - except KeyError: - msg = "Invalid \\escape: " + repr(esc) - raise JSONDecodeError(msg, s, end) - end += 1 - else: - # Unicode escape sequence - esc = s[end + 1:end + 5] - next_end = end + 5 - if len(esc) != 4: - msg = "Invalid \\uXXXX escape" - raise JSONDecodeError(msg, s, end) - uni = int(esc, 16) - # Check for surrogate pair on UCS-4 systems - if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: - msg = "Invalid \\uXXXX\\uXXXX surrogate pair" - if not s[end + 5:end + 7] == '\\u': - raise JSONDecodeError(msg, s, end) - esc2 = s[end + 7:end + 11] - if len(esc2) != 4: - raise JSONDecodeError(msg, s, end) - uni2 = int(esc2, 16) - uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) - next_end += 6 - char = unichr(uni) - end = next_end - # Append the unescaped character - _append(char) - return u''.join(chunks), end - - -# Use speedup if available -scanstring = c_scanstring or py_scanstring - -WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) -WHITESPACE_STR = ' \t\n\r' - -def JSONObject((s, end), encoding, strict, scan_once, object_hook, - object_pairs_hook, memo=None, - _w=WHITESPACE.match, _ws=WHITESPACE_STR): - # Backwards compatibility - if memo is None: - memo = {} - memo_get = memo.setdefault - pairs = [] - # Use a slice to prevent IndexError from being raised, the following - # check will raise a more specific ValueError if the string is empty - nextchar = s[end:end + 1] - # Normally we expect nextchar == '"' - if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() - nextchar = s[end:end + 1] - # Trivial empty object - if nextchar == '}': - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = {} - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end + 1 - elif nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end) - end += 1 - while True: - key, end = scanstring(s, end, encoding, strict) - key = memo_get(key, key) - - # To skip some function call overhead we optimize the fast paths where - # the JSON key separator is ": " or just ":". - if s[end:end + 1] != ':': - end = _w(s, end).end() - if s[end:end + 1] != ':': - raise JSONDecodeError("Expecting : delimiter", s, end) - - end += 1 - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - pairs.append((key, value)) - - try: - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - end += 1 - - if nextchar == '}': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end - 1) - - try: - nextchar = s[end] - if nextchar in _ws: - end += 1 - nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end] - except IndexError: - nextchar = '' - - end += 1 - if nextchar != '"': - raise JSONDecodeError("Expecting property name", s, end - 1) - - if object_pairs_hook is not None: - result = object_pairs_hook(pairs) - return result, end - pairs = dict(pairs) - if object_hook is not None: - pairs = object_hook(pairs) - return pairs, end - -def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): - values = [] - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - # Look-ahead for trivial empty array - if nextchar == ']': - return values, end + 1 - _append = values.append - while True: - try: - value, end = scan_once(s, end) - except StopIteration: - raise JSONDecodeError("Expecting object", s, end) - _append(value) - nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() - nextchar = s[end:end + 1] - end += 1 - if nextchar == ']': - break - elif nextchar != ',': - raise JSONDecodeError("Expecting , delimiter", s, end) - - try: - if s[end] in _ws: - end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() - except IndexError: - pass - - return values, end - -class JSONDecoder(object): - """Simple JSON <http://json.org> decoder - - Performs the following translations in decoding by default: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | unicode | - +---------------+-------------------+ - | number (int) | int, long | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as - their corresponding ``float`` values, which is outside the JSON spec. - - """ - - def __init__(self, encoding=None, object_hook=None, parse_float=None, - parse_int=None, parse_constant=None, strict=True, - object_pairs_hook=None): - """ - *encoding* determines the encoding used to interpret any - :class:`str` objects decoded by this instance (``'utf-8'`` by - default). It has no effect when decoding :class:`unicode` objects. - - Note that currently only encodings that are a superset of ASCII work, - strings of other encodings should be passed in as :class:`unicode`. - - *object_hook*, if specified, will be called with the result of every - JSON object decoded and its return value will be used in place of the - given :class:`dict`. This can be used to provide custom - deserializations (e.g. to support JSON-RPC class hinting). - - *object_pairs_hook* is an optional function that will be called with - the result of any object literal decode with an ordered list of pairs. - The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders - that rely on the order that the key and value pairs are decoded (for - example, :func:`collections.OrderedDict` will remember the order of - insertion). If *object_hook* is also defined, the *object_pairs_hook* - takes priority. - - *parse_float*, if specified, will be called with the string of every - JSON float to be decoded. By default, this is equivalent to - ``float(num_str)``. This can be used to use another datatype or parser - for JSON floats (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every - JSON int to be decoded. By default, this is equivalent to - ``int(num_str)``. This can be used to use another datatype or parser - for JSON integers (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the - following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This - can be used to raise an exception if invalid JSON numbers are - encountered. - - *strict* controls the parser's behavior when it encounters an - invalid control character in a string. The default setting of - ``True`` means that unescaped control characters are parse errors, if - ``False`` then control characters will be allowed in strings. - - """ - self.encoding = encoding - self.object_hook = object_hook - self.object_pairs_hook = object_pairs_hook - self.parse_float = parse_float or float - self.parse_int = parse_int or int - self.parse_constant = parse_constant or _CONSTANTS.__getitem__ - self.strict = strict - self.parse_object = JSONObject - self.parse_array = JSONArray - self.parse_string = scanstring - self.memo = {} - self.scan_once = make_scanner(self) - - def decode(self, s, _w=WHITESPACE.match): - """Return the Python representation of ``s`` (a ``str`` or ``unicode`` - instance containing a JSON document) - - """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() - if end != len(s): - raise JSONDecodeError("Extra data", s, end, len(s)) - return obj - - def raw_decode(self, s, idx=0): - """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` - beginning with a JSON document) and return a 2-tuple of the Python - representation and the index in ``s`` where the document ended. - - This can be used to decode a JSON document from a string that may - have extraneous data at the end. - - """ - try: - obj, end = self.scan_once(s, idx) - except StopIteration: - raise JSONDecodeError("No JSON object could be decoded", s, idx) - return obj, end diff --git a/tablib/packages/simplejson/encoder.py b/tablib/packages/simplejson/encoder.py deleted file mode 100644 index cab8456..0000000 --- a/tablib/packages/simplejson/encoder.py +++ /dev/null @@ -1,501 +0,0 @@ -"""Implementation of JSONEncoder -""" -import re -from decimal import Decimal - -def _import_speedups(): - try: - from simplejson import _speedups - return _speedups.encode_basestring_ascii, _speedups.make_encoder - except ImportError: - return None, None -c_encode_basestring_ascii, c_make_encoder = _import_speedups() - -from simplejson.decoder import PosInf - -ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -HAS_UTF8 = re.compile(r'[\x80-\xff]') -ESCAPE_DCT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} -for i in range(0x20): - #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) - ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) - -FLOAT_REPR = repr - -def encode_basestring(s): - """Return a JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - return ESCAPE_DCT[match.group(0)] - return u'"' + ESCAPE.sub(replace, s) + u'"' - - -def py_encode_basestring_ascii(s): - """Return an ASCII-only JSON representation of a Python string - - """ - if isinstance(s, str) and HAS_UTF8.search(s) is not None: - s = s.decode('utf-8') - def replace(match): - s = match.group(0) - try: - return ESCAPE_DCT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - #return '\\u{0:04x}'.format(n) - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -encode_basestring_ascii = ( - c_encode_basestring_ascii or py_encode_basestring_ascii) - -class JSONEncoder(object): - """Extensible JSON <http://json.org> encoder for Python data structures. - - Supports the following objects and types by default: - - +-------------------+---------------+ - | Python | JSON | - +===================+===============+ - | dict | object | - +-------------------+---------------+ - | list, tuple | array | - +-------------------+---------------+ - | str, unicode | string | - +-------------------+---------------+ - | int, long, float | number | - +-------------------+---------------+ - | True | true | - +-------------------+---------------+ - | False | false | - +-------------------+---------------+ - | None | null | - +-------------------+---------------+ - - To extend this to recognize other objects, subclass and implement a - ``.default()`` method with another method that returns a serializable - object for ``o`` if possible, otherwise it should call the superclass - implementation (to raise ``TypeError``). - - """ - item_separator = ', ' - key_separator = ': ' - def __init__(self, skipkeys=False, ensure_ascii=True, - check_circular=True, allow_nan=True, sort_keys=False, - indent=None, separators=None, encoding='utf-8', default=None, - use_decimal=False): - """Constructor for JSONEncoder, with sensible defaults. - - If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, long, float or None. If - skipkeys is True, such items are simply skipped. - - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming unicode characters escaped. If - ensure_ascii is false, the output will be unicode object. - - If check_circular is true, then lists, dicts, and custom encoded - objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an OverflowError). - Otherwise, no such check takes place. - - If allow_nan is true, then NaN, Infinity, and -Infinity will be - encoded as such. This behavior is not JSON specification compliant, - but is consistent with most JavaScript based encoders and decoders. - Otherwise, it will be a ValueError to encode such floats. - - If sort_keys is true, then the output of dictionaries will be - sorted by key; this is useful for regression tests to ensure - that JSON serializations can be compared on a day-to-day basis. - - If indent is a string, then JSON array elements and object members - will be pretty-printed with a newline followed by that string repeated - for each level of nesting. ``None`` (the default) selects the most compact - representation without any newlines. For backwards compatibility with - versions of simplejson earlier than 2.1.0, an integer is also accepted - and is converted to a string with that many spaces. - - If specified, separators should be a (item_separator, key_separator) - tuple. The default is (', ', ': '). To get the most compact JSON - representation you should specify (',', ':') to eliminate whitespace. - - If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. - - If encoding is not None, then all input strings will be - transformed into unicode using that encoding prior to JSON-encoding. - The default is UTF-8. - - If use_decimal is true (not the default), ``decimal.Decimal`` will - be supported directly by the encoder. For the inverse, decode JSON - with ``parse_float=decimal.Decimal``. - - """ - - self.skipkeys = skipkeys - self.ensure_ascii = ensure_ascii - self.check_circular = check_circular - self.allow_nan = allow_nan - self.sort_keys = sort_keys - self.use_decimal = use_decimal - if isinstance(indent, (int, long)): - indent = ' ' * indent - self.indent = indent - if separators is not None: - self.item_separator, self.key_separator = separators - if default is not None: - self.default = default - self.encoding = encoding - - def default(self, o): - """Implement this method in a subclass such that it returns - a serializable object for ``o``, or calls the base implementation - (to raise a ``TypeError``). - - For example, to support arbitrary iterators, you could - implement default like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - return JSONEncoder.default(self, o) - - """ - raise TypeError(repr(o) + " is not JSON serializable") - - def encode(self, o): - """Return a JSON string representation of a Python data structure. - - >>> from simplejson import JSONEncoder - >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - """ - # This is for extremely simple cases and benchmarks. - if isinstance(o, basestring): - if isinstance(o, str): - _encoding = self.encoding - if (_encoding is not None - and not (_encoding == 'utf-8')): - o = o.decode(_encoding) - if self.ensure_ascii: - return encode_basestring_ascii(o) - else: - return encode_basestring(o) - # This doesn't pass the iterator directly to ''.join() because the - # exceptions aren't as detailed. The list call should be roughly - # equivalent to the PySequence_Fast that ''.join() would do. - chunks = self.iterencode(o, _one_shot=True) - if not isinstance(chunks, (list, tuple)): - chunks = list(chunks) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - """Encode the given object and yield each string - representation as available. - - For example:: - - for chunk in JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - """ - if self.check_circular: - markers = {} - else: - markers = None - if self.ensure_ascii: - _encoder = encode_basestring_ascii - else: - _encoder = encode_basestring - if self.encoding != 'utf-8': - def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): - if isinstance(o, str): - o = o.decode(_encoding) - return _orig_encoder(o) - - def floatstr(o, allow_nan=self.allow_nan, - _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): - # Check for specials. Note that this type of test is processor - # and/or platform-specific, so do tests which don't depend on - # the internals. - - if o != o: - text = 'NaN' - elif o == _inf: - text = 'Infinity' - elif o == _neginf: - text = '-Infinity' - else: - return _repr(o) - - if not allow_nan: - raise ValueError( - "Out of range float values are not JSON compliant: " + - repr(o)) - - return text - - - key_memo = {} - if (_one_shot and c_make_encoder is not None - and not self.indent and not self.sort_keys): - _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, self.allow_nan, key_memo, self.use_decimal) - else: - _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, - self.key_separator, self.item_separator, self.sort_keys, - self.skipkeys, _one_shot, self.use_decimal) - try: - return _iterencode(o, 0) - finally: - key_memo.clear() - - -class JSONEncoderForHTML(JSONEncoder): - """An encoder that produces JSON safe to embed in HTML. - - To embed JSON content in, say, a script tag on a web page, the - characters &, < and > should be escaped. They cannot be escaped - with the usual entities (e.g. &) because they are not expanded - within <script> tags. - """ - - def encode(self, o): - # Override JSONEncoder.encode because it has hacks for - # performance that make things more complicated. - chunks = self.iterencode(o, True) - if self.ensure_ascii: - return ''.join(chunks) - else: - return u''.join(chunks) - - def iterencode(self, o, _one_shot=False): - chunks = super(JSONEncoderForHTML, self).iterencode(o, _one_shot) - for chunk in chunks: - chunk = chunk.replace('&', '\\u0026') - chunk = chunk.replace('<', '\\u003c') - chunk = chunk.replace('>', '\\u003e') - yield chunk - - -def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, - _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - _use_decimal, - ## HACK: hand-optimized bytecode; turn globals into locals - False=False, - True=True, - ValueError=ValueError, - basestring=basestring, - Decimal=Decimal, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - long=long, - str=str, - tuple=tuple, - ): - - def _iterencode_list(lst, _current_indent_level): - if not lst: - yield '[]' - return - if markers is not None: - markerid = id(lst) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = lst - buf = '[' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - separator = _item_separator + newline_indent - buf += newline_indent - else: - newline_indent = None - separator = _item_separator - first = True - for value in lst: - if first: - first = False - else: - buf = separator - if isinstance(value, basestring): - yield buf + _encoder(value) - elif value is None: - yield buf + 'null' - elif value is True: - yield buf + 'true' - elif value is False: - yield buf + 'false' - elif isinstance(value, (int, long)): - yield buf + str(value) - elif isinstance(value, float): - yield buf + _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield buf + str(value) - else: - yield buf - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield ']' - if markers is not None: - del markers[markerid] - - def _iterencode_dict(dct, _current_indent_level): - if not dct: - yield '{}' - return - if markers is not None: - markerid = id(dct) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = dct - yield '{' - if _indent is not None: - _current_indent_level += 1 - newline_indent = '\n' + (_indent * _current_indent_level) - item_separator = _item_separator + newline_indent - yield newline_indent - else: - newline_indent = None - item_separator = _item_separator - first = True - if _sort_keys: - items = dct.items() - items.sort(key=lambda kv: kv[0]) - else: - items = dct.iteritems() - for key, value in items: - if isinstance(key, basestring): - pass - # JavaScript is weakly typed for these, so it makes sense to - # also allow them. Many encoders seem to do something like this. - elif isinstance(key, float): - key = _floatstr(key) - elif key is True: - key = 'true' - elif key is False: - key = 'false' - elif key is None: - key = 'null' - elif isinstance(key, (int, long)): - key = str(key) - elif _skipkeys: - continue - else: - raise TypeError("key " + repr(key) + " is not a string") - if first: - first = False - else: - yield item_separator - yield _encoder(key) - yield _key_separator - if isinstance(value, basestring): - yield _encoder(value) - elif value is None: - yield 'null' - elif value is True: - yield 'true' - elif value is False: - yield 'false' - elif isinstance(value, (int, long)): - yield str(value) - elif isinstance(value, float): - yield _floatstr(value) - elif _use_decimal and isinstance(value, Decimal): - yield str(value) - else: - if isinstance(value, (list, tuple)): - chunks = _iterencode_list(value, _current_indent_level) - elif isinstance(value, dict): - chunks = _iterencode_dict(value, _current_indent_level) - else: - chunks = _iterencode(value, _current_indent_level) - for chunk in chunks: - yield chunk - if newline_indent is not None: - _current_indent_level -= 1 - yield '\n' + (_indent * _current_indent_level) - yield '}' - if markers is not None: - del markers[markerid] - - def _iterencode(o, _current_indent_level): - if isinstance(o, basestring): - yield _encoder(o) - elif o is None: - yield 'null' - elif o is True: - yield 'true' - elif o is False: - yield 'false' - elif isinstance(o, (int, long)): - yield str(o) - elif isinstance(o, float): - yield _floatstr(o) - elif isinstance(o, (list, tuple)): - for chunk in _iterencode_list(o, _current_indent_level): - yield chunk - elif isinstance(o, dict): - for chunk in _iterencode_dict(o, _current_indent_level): - yield chunk - elif _use_decimal and isinstance(o, Decimal): - yield str(o) - else: - if markers is not None: - markerid = id(o) - if markerid in markers: - raise ValueError("Circular reference detected") - markers[markerid] = o - o = _default(o) - for chunk in _iterencode(o, _current_indent_level): - yield chunk - if markers is not None: - del markers[markerid] - - return _iterencode diff --git a/tablib/packages/simplejson/ordered_dict.py b/tablib/packages/simplejson/ordered_dict.py deleted file mode 100644 index 87ad888..0000000 --- a/tablib/packages/simplejson/ordered_dict.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Drop-in replacement for collections.OrderedDict by Raymond Hettinger - -http://code.activestate.com/recipes/576693/ - -""" -from UserDict import DictMixin - -# Modified from original to support Python 2.4, see -# http://code.google.com/p/simplejson/issues/detail?id=53 -try: - all -except NameError: - def all(seq): - for elem in seq: - if not elem: - return False - return True - -class OrderedDict(dict, DictMixin): - - def __init__(self, *args, **kwds): - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__end - except AttributeError: - self.clear() - self.update(*args, **kwds) - - def clear(self): - self.__end = end = [] - end += [None, end, end] # sentinel node for doubly linked list - self.__map = {} # key --> [key, prev, next] - dict.clear(self) - - def __setitem__(self, key, value): - if key not in self: - end = self.__end - curr = end[1] - curr[2] = end[1] = self.__map[key] = [key, curr, end] - dict.__setitem__(self, key, value) - - def __delitem__(self, key): - dict.__delitem__(self, key) - key, prev, next = self.__map.pop(key) - prev[2] = next - next[1] = prev - - def __iter__(self): - end = self.__end - curr = end[2] - while curr is not end: - yield curr[0] - curr = curr[2] - - def __reversed__(self): - end = self.__end - curr = end[1] - while curr is not end: - yield curr[0] - curr = curr[1] - - def popitem(self, last=True): - if not self: - raise KeyError('dictionary is empty') - # Modified from original to support Python 2.4, see - # http://code.google.com/p/simplejson/issues/detail?id=53 - if last: - key = reversed(self).next() - else: - key = iter(self).next() - value = self.pop(key) - return key, value - - def __reduce__(self): - items = [[k, self[k]] for k in self] - tmp = self.__map, self.__end - del self.__map, self.__end - inst_dict = vars(self).copy() - self.__map, self.__end = tmp - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def keys(self): - return list(self) - - setdefault = DictMixin.setdefault - update = DictMixin.update - pop = DictMixin.pop - values = DictMixin.values - items = DictMixin.items - iterkeys = DictMixin.iterkeys - itervalues = DictMixin.itervalues - iteritems = DictMixin.iteritems - - def __repr__(self): - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - - def copy(self): - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - if isinstance(other, OrderedDict): - return len(self)==len(other) and \ - all(p==q for p, q in zip(self.items(), other.items())) - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other diff --git a/tablib/packages/simplejson/scanner.py b/tablib/packages/simplejson/scanner.py deleted file mode 100644 index 54593a3..0000000 --- a/tablib/packages/simplejson/scanner.py +++ /dev/null @@ -1,77 +0,0 @@ -"""JSON token scanner -""" -import re -def _import_c_make_scanner(): - try: - from simplejson._speedups import make_scanner - return make_scanner - except ImportError: - return None -c_make_scanner = _import_c_make_scanner() - -__all__ = ['make_scanner'] - -NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', - (re.VERBOSE | re.MULTILINE | re.DOTALL)) - -def py_make_scanner(context): - parse_object = context.parse_object - parse_array = context.parse_array - parse_string = context.parse_string - match_number = NUMBER_RE.match - encoding = context.encoding - strict = context.strict - parse_float = context.parse_float - parse_int = context.parse_int - parse_constant = context.parse_constant - object_hook = context.object_hook - object_pairs_hook = context.object_pairs_hook - memo = context.memo - - def _scan_once(string, idx): - try: - nextchar = string[idx] - except IndexError: - raise StopIteration - - if nextchar == '"': - return parse_string(string, idx + 1, encoding, strict) - elif nextchar == '{': - return parse_object((string, idx + 1), encoding, strict, - _scan_once, object_hook, object_pairs_hook, memo) - elif nextchar == '[': - return parse_array((string, idx + 1), _scan_once) - elif nextchar == 'n' and string[idx:idx + 4] == 'null': - return None, idx + 4 - elif nextchar == 't' and string[idx:idx + 4] == 'true': - return True, idx + 4 - elif nextchar == 'f' and string[idx:idx + 5] == 'false': - return False, idx + 5 - - m = match_number(string, idx) - if m is not None: - integer, frac, exp = m.groups() - if frac or exp: - res = parse_float(integer + (frac or '') + (exp or '')) - else: - res = parse_int(integer) - return res, m.end() - elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': - return parse_constant('NaN'), idx + 3 - elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': - return parse_constant('Infinity'), idx + 8 - elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': - return parse_constant('-Infinity'), idx + 9 - else: - raise StopIteration - - def scan_once(string, idx): - try: - return _scan_once(string, idx) - finally: - memo.clear() - - return scan_once - -make_scanner = c_make_scanner or py_make_scanner diff --git a/tablib/packages/simplejson/tests/__init__.py b/tablib/packages/simplejson/tests/__init__.py deleted file mode 100644 index 22d8465..0000000 --- a/tablib/packages/simplejson/tests/__init__.py +++ /dev/null @@ -1,63 +0,0 @@ -import unittest -import doctest - - -class OptionalExtensionTestSuite(unittest.TestSuite): - def run(self, result): - import simplejson - run = unittest.TestSuite.run - run(self, result) - simplejson._toggle_speedups(False) - run(self, result) - simplejson._toggle_speedups(True) - return result - - -def additional_tests(suite=None): - import simplejson - import simplejson.encoder - import simplejson.decoder - if suite is None: - suite = unittest.TestSuite() - for mod in (simplejson, simplejson.encoder, simplejson.decoder): - suite.addTest(doctest.DocTestSuite(mod)) - suite.addTest(doctest.DocFileSuite('../../index.rst')) - return suite - - -def all_tests_suite(): - suite = unittest.TestLoader().loadTestsFromNames([ - 'simplejson.tests.test_check_circular', - 'simplejson.tests.test_decode', - 'simplejson.tests.test_default', - 'simplejson.tests.test_dump', - 'simplejson.tests.test_encode_basestring_ascii', - 'simplejson.tests.test_encode_for_html', - 'simplejson.tests.test_fail', - 'simplejson.tests.test_float', - 'simplejson.tests.test_indent', - 'simplejson.tests.test_pass1', - 'simplejson.tests.test_pass2', - 'simplejson.tests.test_pass3', - 'simplejson.tests.test_recursion', - 'simplejson.tests.test_scanstring', - 'simplejson.tests.test_separators', - 'simplejson.tests.test_speedups', - 'simplejson.tests.test_unicode', - 'simplejson.tests.test_decimal', - ]) - suite = additional_tests(suite) - return OptionalExtensionTestSuite([suite]) - - -def main(): - runner = unittest.TextTestRunner() - suite = all_tests_suite() - runner.run(suite) - - -if __name__ == '__main__': - import os - import sys - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - main() diff --git a/tablib/packages/simplejson/tests/test_check_circular.py b/tablib/packages/simplejson/tests/test_check_circular.py deleted file mode 100644 index af6463d..0000000 --- a/tablib/packages/simplejson/tests/test_check_circular.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest import TestCase -import simplejson as json - -def default_iterable(obj): - return list(obj) - -class TestCheckCircular(TestCase): - def test_circular_dict(self): - dct = {} - dct['a'] = dct - self.assertRaises(ValueError, json.dumps, dct) - - def test_circular_list(self): - lst = [] - lst.append(lst) - self.assertRaises(ValueError, json.dumps, lst) - - def test_circular_composite(self): - dct2 = {} - dct2['a'] = [] - dct2['a'].append(dct2) - self.assertRaises(ValueError, json.dumps, dct2) - - def test_circular_default(self): - json.dumps([set()], default=default_iterable) - self.assertRaises(TypeError, json.dumps, [set()]) - - def test_circular_off_default(self): - json.dumps([set()], default=default_iterable, check_circular=False) - self.assertRaises(TypeError, json.dumps, [set()], check_circular=False) diff --git a/tablib/packages/simplejson/tests/test_decimal.py b/tablib/packages/simplejson/tests/test_decimal.py deleted file mode 100644 index a694c7e..0000000 --- a/tablib/packages/simplejson/tests/test_decimal.py +++ /dev/null @@ -1,33 +0,0 @@ -from decimal import Decimal -from unittest import TestCase - -import simplejson as json - -class TestDecimal(TestCase): - NUMS = "1.0", "10.00", "1.1", "1234567890.1234567890", "500" - def test_decimal_encode(self): - for d in map(Decimal, self.NUMS): - self.assertEquals(json.dumps(d, use_decimal=True), str(d)) - - def test_decimal_decode(self): - for s in self.NUMS: - self.assertEquals(json.loads(s, parse_float=Decimal), Decimal(s)) - - def test_decimal_roundtrip(self): - for d in map(Decimal, self.NUMS): - # The type might not be the same (int and Decimal) but they - # should still compare equal. - self.assertEquals( - json.loads( - json.dumps(d, use_decimal=True), parse_float=Decimal), - d) - self.assertEquals( - json.loads( - json.dumps([d], use_decimal=True), parse_float=Decimal), - [d]) - - def test_decimal_defaults(self): - d = Decimal(1) - # use_decimal=False is the default - self.assertRaises(TypeError, json.dumps, d, use_decimal=False) - self.assertRaises(TypeError, json.dumps, d)
\ No newline at end of file diff --git a/tablib/packages/simplejson/tests/test_decode.py b/tablib/packages/simplejson/tests/test_decode.py deleted file mode 100644 index a5a1c48..0000000 --- a/tablib/packages/simplejson/tests/test_decode.py +++ /dev/null @@ -1,73 +0,0 @@ -import decimal -from unittest import TestCase -from StringIO import StringIO - -import simplejson as json -from simplejson import OrderedDict - -class TestDecode(TestCase): - if not hasattr(TestCase, 'assertIs'): - def assertIs(self, a, b): - self.assertTrue(a is b, '%r is %r' % (a, b)) - - def test_decimal(self): - rval = json.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) - self.assertEquals(rval, decimal.Decimal('1.1')) - - def test_float(self): - rval = json.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) - self.assertEquals(rval, 1.0) - - def test_decoder_optimizations(self): - # Several optimizations were made that skip over calls to - # the whitespace regex, so this test is designed to try and - # exercise the uncommon cases. The array cases are already covered. - rval = json.loads('{ "key" : "value" , "k":"v" }') - self.assertEquals(rval, {"key":"value", "k":"v"}) - - def test_empty_objects(self): - s = '{}' - self.assertEqual(json.loads(s), eval(s)) - s = '[]' - self.assertEqual(json.loads(s), eval(s)) - s = '""' - self.assertEqual(json.loads(s), eval(s)) - - def test_object_pairs_hook(self): - s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), - ("qrt", 5), ("pad", 6), ("hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - self.assertEqual(json.load(StringIO(s), - object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=OrderedDict) - self.assertEqual(od, OrderedDict(p)) - self.assertEqual(type(od), OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=OrderedDict, - object_hook=lambda x: None), - OrderedDict(p)) - - def check_keys_reuse(self, source, loads): - rval = loads(source) - (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) - self.assertIs(a, c) - self.assertIs(b, d) - - def test_keys_reuse_str(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'.encode('utf8') - self.check_keys_reuse(s, json.loads) - - def test_keys_reuse_unicode(self): - s = u'[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' - self.check_keys_reuse(s, json.loads) - - def test_empty_strings(self): - self.assertEqual(json.loads('""'), "") - self.assertEqual(json.loads(u'""'), u"") - self.assertEqual(json.loads('[""]'), [""]) - self.assertEqual(json.loads(u'[""]'), [u""]) diff --git a/tablib/packages/simplejson/tests/test_default.py b/tablib/packages/simplejson/tests/test_default.py deleted file mode 100644 index 139e42b..0000000 --- a/tablib/packages/simplejson/tests/test_default.py +++ /dev/null @@ -1,9 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class TestDefault(TestCase): - def test_default(self): - self.assertEquals( - json.dumps(type, default=repr), - json.dumps(repr(type))) diff --git a/tablib/packages/simplejson/tests/test_dump.py b/tablib/packages/simplejson/tests/test_dump.py deleted file mode 100644 index 8b34004..0000000 --- a/tablib/packages/simplejson/tests/test_dump.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest import TestCase -from cStringIO import StringIO - -import simplejson as json - -class TestDump(TestCase): - def test_dump(self): - sio = StringIO() - json.dump({}, sio) - self.assertEquals(sio.getvalue(), '{}') - - def test_dumps(self): - self.assertEquals(json.dumps({}), '{}') - - def test_encode_truefalse(self): - self.assertEquals(json.dumps( - {True: False, False: True}, sort_keys=True), - '{"false": true, "true": false}') - self.assertEquals(json.dumps( - {2: 3.0, 4.0: 5L, False: 1, 6L: True, "7": 0}, sort_keys=True), - '{"false": 1, "2": 3.0, "4.0": 5, "6": true, "7": 0}') - - def test_ordered_dict(self): - # http://bugs.python.org/issue6105 - items = [('one', 1), ('two', 2), ('three', 3), ('four', 4), ('five', 5)] - s = json.dumps(json.OrderedDict(items)) - self.assertEqual(s, '{"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}')
\ No newline at end of file diff --git a/tablib/packages/simplejson/tests/test_encode_basestring_ascii.py b/tablib/packages/simplejson/tests/test_encode_basestring_ascii.py deleted file mode 100644 index b19631a..0000000 --- a/tablib/packages/simplejson/tests/test_encode_basestring_ascii.py +++ /dev/null @@ -1,41 +0,0 @@ -from unittest import TestCase - -import simplejson.encoder - -CASES = [ - (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), - (u'controls', '"controls"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), - (u' s p a c e d ', '" s p a c e d "'), - (u'\U0001d120', '"\\ud834\\udd20"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - ('\xce\xb1\xce\xa9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - ('\xce\xb1\xce\xa9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), - (u"`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), -] - -class TestEncodeBaseStringAscii(TestCase): - def test_py_encode_basestring_ascii(self): - self._test_encode_basestring_ascii(simplejson.encoder.py_encode_basestring_ascii) - - def test_c_encode_basestring_ascii(self): - if not simplejson.encoder.c_encode_basestring_ascii: - return - self._test_encode_basestring_ascii(simplejson.encoder.c_encode_basestring_ascii) - - def _test_encode_basestring_ascii(self, encode_basestring_ascii): - fname = encode_basestring_ascii.__name__ - for input_string, expect in CASES: - result = encode_basestring_ascii(input_string) - #self.assertEquals(result, expect, - # '{0!r} != {1!r} for {2}({3!r})'.format( - # result, expect, fname, input_string)) - self.assertEquals(result, expect, - '%r != %r for %s(%r)' % (result, expect, fname, input_string)) diff --git a/tablib/packages/simplejson/tests/test_encode_for_html.py b/tablib/packages/simplejson/tests/test_encode_for_html.py deleted file mode 100644 index c2d5f80..0000000 --- a/tablib/packages/simplejson/tests/test_encode_for_html.py +++ /dev/null @@ -1,32 +0,0 @@ -import unittest - -import simplejson.decoder -import simplejson.encoder - - -class TestEncodeForHTML(unittest.TestCase): - - def setUp(self): - self.decoder = simplejson.decoder.JSONDecoder() - self.encoder = simplejson.encoder.JSONEncoderForHTML() - - def test_basic_encode(self): - self.assertEqual(r'"\u0026"', self.encoder.encode('&')) - self.assertEqual(r'"\u003c"', self.encoder.encode('<')) - self.assertEqual(r'"\u003e"', self.encoder.encode('>')) - - def test_basic_roundtrip(self): - for char in '&<>': - self.assertEqual( - char, self.decoder.decode( - self.encoder.encode(char))) - - def test_prevent_script_breakout(self): - bad_string = '</script><script>alert("gotcha")</script>' - self.assertEqual( - r'"\u003c/script\u003e\u003cscript\u003e' - r'alert(\"gotcha\")\u003c/script\u003e"', - self.encoder.encode(bad_string)) - self.assertEqual( - bad_string, self.decoder.decode( - self.encoder.encode(bad_string))) diff --git a/tablib/packages/simplejson/tests/test_fail.py b/tablib/packages/simplejson/tests/test_fail.py deleted file mode 100644 index 646c0f4..0000000 --- a/tablib/packages/simplejson/tests/test_fail.py +++ /dev/null @@ -1,91 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# Fri Dec 30 18:57:26 2005 -JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json - '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json - '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json - '{unquoted_key: "keys must be quoted}', - # http://json.org/JSON_checker/test/fail4.json - '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json - '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json - '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json - '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json - '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json - '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json - '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json - '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json - '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json - '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json - '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json - '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json - '["Illegal backslash escape: \\\'"]', - # http://json.org/JSON_checker/test/fail17.json - '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json - '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json - '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json - '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json - '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json - '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json - '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json - "['single quote']", - # http://code.google.com/p/simplejson/issues/detail?id=3 - u'["A\u001FZ control characters in string"]', -] - -SKIPS = { - 1: "why not have a string payload?", - 18: "spec doesn't specify any nesting limitations", -} - -class TestFail(TestCase): - def test_failures(self): - for idx, doc in enumerate(JSONDOCS): - idx = idx + 1 - if idx in SKIPS: - json.loads(doc) - continue - try: - json.loads(doc) - except json.JSONDecodeError: - pass - else: - #self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) - self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) - - def test_array_decoder_issue46(self): - # http://code.google.com/p/simplejson/issues/detail?id=46 - for doc in [u'[,]', '[,]']: - try: - json.loads(doc) - except json.JSONDecodeError, e: - self.assertEquals(e.pos, 1) - self.assertEquals(e.lineno, 1) - self.assertEquals(e.colno, 1) - except Exception, e: - self.fail("Unexpected exception raised %r %s" % (e, e)) - else: - self.fail("Unexpected success parsing '[,]'")
\ No newline at end of file diff --git a/tablib/packages/simplejson/tests/test_float.py b/tablib/packages/simplejson/tests/test_float.py deleted file mode 100644 index 94502c6..0000000 --- a/tablib/packages/simplejson/tests/test_float.py +++ /dev/null @@ -1,19 +0,0 @@ -import math -from unittest import TestCase - -import simplejson as json - -class TestFloat(TestCase): - def test_floats(self): - for num in [1617161771.7650001, math.pi, math.pi**100, - math.pi**-100, 3.1]: - self.assertEquals(float(json.dumps(num)), num) - self.assertEquals(json.loads(json.dumps(num)), num) - self.assertEquals(json.loads(unicode(json.dumps(num))), num) - - def test_ints(self): - for num in [1, 1L, 1<<32, 1<<64]: - self.assertEquals(json.dumps(num), str(num)) - self.assertEquals(int(json.dumps(num)), num) - self.assertEquals(json.loads(json.dumps(num)), num) - self.assertEquals(json.loads(unicode(json.dumps(num))), num) diff --git a/tablib/packages/simplejson/tests/test_indent.py b/tablib/packages/simplejson/tests/test_indent.py deleted file mode 100644 index 985831b..0000000 --- a/tablib/packages/simplejson/tests/test_indent.py +++ /dev/null @@ -1,53 +0,0 @@ -from unittest import TestCase - -import simplejson as json -import textwrap - -class TestIndent(TestCase): - def test_indent(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', - 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - \t[ - \t\t"blorpie" - \t], - \t[ - \t\t"whoops" - \t], - \t[], - \t"d-shtaeou", - \t"d-nthiouh", - \t"i-vhbjkhnth", - \t{ - \t\t"nifty": 87 - \t}, - \t{ - \t\t"field": "yes", - \t\t"morefield": false - \t} - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent='\t', sort_keys=True, separators=(',', ': ')) - d3 = json.dumps(h, indent=' ', sort_keys=True, separators=(',', ': ')) - d4 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - h3 = json.loads(d3) - h4 = json.loads(d4) - - self.assertEquals(h1, h) - self.assertEquals(h2, h) - self.assertEquals(h3, h) - self.assertEquals(h4, h) - self.assertEquals(d3, expect.replace('\t', ' ')) - self.assertEquals(d4, expect.replace('\t', ' ')) - # NOTE: Python 2.4 textwrap.dedent converts tabs to spaces, - # so the following is expected to fail. Python 2.4 is not a - # supported platform in simplejson 2.1.0+. - self.assertEquals(d2, expect) diff --git a/tablib/packages/simplejson/tests/test_pass1.py b/tablib/packages/simplejson/tests/test_pass1.py deleted file mode 100644 index c3d6302..0000000 --- a/tablib/packages/simplejson/tests/test_pass1.py +++ /dev/null @@ -1,76 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass1.json -JSON = r''' -[ - "JSON Test Pattern pass1", - {"object with 1 member":["array with 1 element"]}, - {}, - [], - -42, - true, - false, - null, - { - "integer": 1234567890, - "real": -9876.543210, - "e": 0.123456789e-12, - "E": 1.234567890E+34, - "": 23456789012E666, - "zero": 0, - "one": 1, - "space": " ", - "quote": "\"", - "backslash": "\\", - "controls": "\b\f\n\r\t", - "slash": "/ & \/", - "alpha": "abcdefghijklmnopqrstuvwyz", - "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", - "digit": "0123456789", - "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?", - "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", - "true": true, - "false": false, - "null": null, - "array":[ ], - "object":{ }, - "address": "50 St. James Street", - "url": "http://www.JSON.org/", - "comment": "// /* <!-- --", - "# -- --> */": " ", - " s p a c e d " :[1,2 , 3 - -, - -4 , 5 , 6 ,7 ], - "compact": [1,2,3,4,5,6,7], - "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", - "quotes": "" \u0022 %22 0x22 034 "", - "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" -: "A key can be any string" - }, - 0.5 ,98.6 -, -99.44 -, - -1066 - - -,"rosebud"] -''' - -class TestPass1(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEquals(res, json.loads(out)) - try: - json.dumps(res, allow_nan=False) - except ValueError: - pass - else: - self.fail("23456789012E666 should be out of range") diff --git a/tablib/packages/simplejson/tests/test_pass2.py b/tablib/packages/simplejson/tests/test_pass2.py deleted file mode 100644 index de4ee00..0000000 --- a/tablib/packages/simplejson/tests/test_pass2.py +++ /dev/null @@ -1,14 +0,0 @@ -from unittest import TestCase -import simplejson as json - -# from http://json.org/JSON_checker/test/pass2.json -JSON = r''' -[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] -''' - -class TestPass2(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEquals(res, json.loads(out)) diff --git a/tablib/packages/simplejson/tests/test_pass3.py b/tablib/packages/simplejson/tests/test_pass3.py deleted file mode 100644 index f591aba..0000000 --- a/tablib/packages/simplejson/tests/test_pass3.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -# from http://json.org/JSON_checker/test/pass3.json -JSON = r''' -{ - "JSON Test Pattern pass3": { - "The outermost value": "must be an object or array.", - "In this test": "It is an object." - } -} -''' - -class TestPass3(TestCase): - def test_parse(self): - # test in/out equivalence and parsing - res = json.loads(JSON) - out = json.dumps(res) - self.assertEquals(res, json.loads(out)) diff --git a/tablib/packages/simplejson/tests/test_recursion.py b/tablib/packages/simplejson/tests/test_recursion.py deleted file mode 100644 index 97422a6..0000000 --- a/tablib/packages/simplejson/tests/test_recursion.py +++ /dev/null @@ -1,67 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class JSONTestObject: - pass - - -class RecursiveJSONEncoder(json.JSONEncoder): - recurse = False - def default(self, o): - if o is JSONTestObject: - if self.recurse: - return [JSONTestObject] - else: - return 'JSONTestObject' - return json.JSONEncoder.default(o) - - -class TestRecursion(TestCase): - def test_listrecursion(self): - x = [] - x.append(x) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on list recursion") - x = [] - y = [x] - x.append(y) - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on alternating list recursion") - y = [] - x = [y, y] - # ensure that the marker is cleared - json.dumps(x) - - def test_dictrecursion(self): - x = {} - x["test"] = x - try: - json.dumps(x) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on dict recursion") - x = {} - y = {"a": x, "b": x} - # ensure that the marker is cleared - json.dumps(x) - - def test_defaultrecursion(self): - enc = RecursiveJSONEncoder() - self.assertEquals(enc.encode(JSONTestObject), '"JSONTestObject"') - enc.recurse = True - try: - enc.encode(JSONTestObject) - except ValueError: - pass - else: - self.fail("didn't raise ValueError on default recursion") diff --git a/tablib/packages/simplejson/tests/test_scanstring.py b/tablib/packages/simplejson/tests/test_scanstring.py deleted file mode 100644 index a7fcd46..0000000 --- a/tablib/packages/simplejson/tests/test_scanstring.py +++ /dev/null @@ -1,117 +0,0 @@ -import sys -from unittest import TestCase - -import simplejson as json -import simplejson.decoder - -class TestScanString(TestCase): - def test_py_scanstring(self): - self._test_scanstring(simplejson.decoder.py_scanstring) - - def test_c_scanstring(self): - if not simplejson.decoder.c_scanstring: - return - self._test_scanstring(simplejson.decoder.c_scanstring) - - def _test_scanstring(self, scanstring): - self.assertEquals( - scanstring('"z\\ud834\\udd20x"', 1, None, True), - (u'z\U0001d120x', 16)) - - if sys.maxunicode == 65535: - self.assertEquals( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 6)) - else: - self.assertEquals( - scanstring(u'"z\U0001d120x"', 1, None, True), - (u'z\U0001d120x', 5)) - - self.assertEquals( - scanstring('"\\u007b"', 1, None, True), - (u'{', 8)) - - self.assertEquals( - scanstring('"A JSON payload should be an object or array, not a string."', 1, None, True), - (u'A JSON payload should be an object or array, not a string.', 60)) - - self.assertEquals( - scanstring('["Unclosed array"', 2, None, True), - (u'Unclosed array', 17)) - - self.assertEquals( - scanstring('["extra comma",]', 2, None, True), - (u'extra comma', 14)) - - self.assertEquals( - scanstring('["double extra comma",,]', 2, None, True), - (u'double extra comma', 21)) - - self.assertEquals( - scanstring('["Comma after the close"],', 2, None, True), - (u'Comma after the close', 24)) - - self.assertEquals( - scanstring('["Extra close"]]', 2, None, True), - (u'Extra close', 14)) - - self.assertEquals( - scanstring('{"Extra comma": true,}', 2, None, True), - (u'Extra comma', 14)) - - self.assertEquals( - scanstring('{"Extra value after close": true} "misplaced quoted value"', 2, None, True), - (u'Extra value after close', 26)) - - self.assertEquals( - scanstring('{"Illegal expression": 1 + 2}', 2, None, True), - (u'Illegal expression', 21)) - - self.assertEquals( - scanstring('{"Illegal invocation": alert()}', 2, None, True), - (u'Illegal invocation', 21)) - - self.assertEquals( - scanstring('{"Numbers cannot have leading zeroes": 013}', 2, None, True), - (u'Numbers cannot have leading zeroes', 37)) - - self.assertEquals( - scanstring('{"Numbers cannot be hex": 0x14}', 2, None, True), - (u'Numbers cannot be hex', 24)) - - self.assertEquals( - scanstring('[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', 21, None, True), - (u'Too deep', 30)) - - self.assertEquals( - scanstring('{"Missing colon" null}', 2, None, True), - (u'Missing colon', 16)) - - self.assertEquals( - scanstring('{"Double colon":: null}', 2, None, True), - (u'Double colon', 15)) - - self.assertEquals( - scanstring('{"Comma instead of colon", null}', 2, None, True), - (u'Comma instead of colon', 25)) - - self.assertEquals( - scanstring('["Colon instead of comma": false]', 2, None, True), - (u'Colon instead of comma', 25)) - - self.assertEquals( - scanstring('["Bad value", truth]', 2, None, True), - (u'Bad value', 12)) - - def test_issue3623(self): - self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1, - "xxx") - self.assertRaises(UnicodeDecodeError, - json.encoder.encode_basestring_ascii, "xx\xff") - - def test_overflow(self): - # Python 2.5 does not have maxsize - maxsize = getattr(sys, 'maxsize', sys.maxint) - self.assertRaises(OverflowError, json.decoder.scanstring, "xxx", - maxsize + 1) - diff --git a/tablib/packages/simplejson/tests/test_separators.py b/tablib/packages/simplejson/tests/test_separators.py deleted file mode 100644 index cbda93c..0000000 --- a/tablib/packages/simplejson/tests/test_separators.py +++ /dev/null @@ -1,42 +0,0 @@ -import textwrap -from unittest import TestCase - -import simplejson as json - - -class TestSeparators(TestCase): - def test_separators(self): - h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', - {'nifty': 87}, {'field': 'yes', 'morefield': False} ] - - expect = textwrap.dedent("""\ - [ - [ - "blorpie" - ] , - [ - "whoops" - ] , - [] , - "d-shtaeou" , - "d-nthiouh" , - "i-vhbjkhnth" , - { - "nifty" : 87 - } , - { - "field" : "yes" , - "morefield" : false - } - ]""") - - - d1 = json.dumps(h) - d2 = json.dumps(h, indent=' ', sort_keys=True, separators=(' ,', ' : ')) - - h1 = json.loads(d1) - h2 = json.loads(d2) - - self.assertEquals(h1, h) - self.assertEquals(h2, h) - self.assertEquals(d2, expect) diff --git a/tablib/packages/simplejson/tests/test_speedups.py b/tablib/packages/simplejson/tests/test_speedups.py deleted file mode 100644 index 4bf0875..0000000 --- a/tablib/packages/simplejson/tests/test_speedups.py +++ /dev/null @@ -1,21 +0,0 @@ -import decimal -from unittest import TestCase - -from simplejson import decoder, encoder, scanner - -def has_speedups(): - return encoder.c_make_encoder is not None - -class TestDecode(TestCase): - def test_make_scanner(self): - if not has_speedups(): - return - self.assertRaises(AttributeError, scanner.c_make_scanner, 1) - - def test_make_encoder(self): - if not has_speedups(): - return - self.assertRaises(TypeError, encoder.c_make_encoder, - None, - "\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", - None) diff --git a/tablib/packages/simplejson/tests/test_unicode.py b/tablib/packages/simplejson/tests/test_unicode.py deleted file mode 100644 index f73e5bf..0000000 --- a/tablib/packages/simplejson/tests/test_unicode.py +++ /dev/null @@ -1,99 +0,0 @@ -from unittest import TestCase - -import simplejson as json - -class TestUnicode(TestCase): - def test_encoding1(self): - encoder = json.JSONEncoder(encoding='utf-8') - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = encoder.encode(u) - js = encoder.encode(s) - self.assertEquals(ju, js) - - def test_encoding2(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - s = u.encode('utf-8') - ju = json.dumps(u, encoding='utf-8') - js = json.dumps(s, encoding='utf-8') - self.assertEquals(ju, js) - - def test_encoding3(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u) - self.assertEquals(j, '"\\u03b1\\u03a9"') - - def test_encoding4(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u]) - self.assertEquals(j, '["\\u03b1\\u03a9"]') - - def test_encoding5(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps(u, ensure_ascii=False) - self.assertEquals(j, u'"' + u + u'"') - - def test_encoding6(self): - u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' - j = json.dumps([u], ensure_ascii=False) - self.assertEquals(j, u'["' + u + u'"]') - - def test_big_unicode_encode(self): - u = u'\U0001d120' - self.assertEquals(json.dumps(u), '"\\ud834\\udd20"') - self.assertEquals(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') - - def test_big_unicode_decode(self): - u = u'z\U0001d120x' - self.assertEquals(json.loads('"' + u + '"'), u) - self.assertEquals(json.loads('"z\\ud834\\udd20x"'), u) - - def test_unicode_decode(self): - for i in range(0, 0xd7ff): - u = unichr(i) - #s = '"\\u{0:04x}"'.format(i) - s = '"\\u%04x"' % (i,) - self.assertEquals(json.loads(s), u) - - def test_object_pairs_hook_with_unicode(self): - s = u'{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' - p = [(u"xkd", 1), (u"kcw", 2), (u"art", 3), (u"hxm", 4), - (u"qrt", 5), (u"pad", 6), (u"hoy", 7)] - self.assertEqual(json.loads(s), eval(s)) - self.assertEqual(json.loads(s, object_pairs_hook=lambda x: x), p) - od = json.loads(s, object_pairs_hook=json.OrderedDict) - self.assertEqual(od, json.OrderedDict(p)) - self.assertEqual(type(od), json.OrderedDict) - # the object_pairs_hook takes priority over the object_hook - self.assertEqual(json.loads(s, - object_pairs_hook=json.OrderedDict, - object_hook=lambda x: None), - json.OrderedDict(p)) - - - def test_default_encoding(self): - self.assertEquals(json.loads(u'{"a": "\xe9"}'.encode('utf-8')), - {'a': u'\xe9'}) - - def test_unicode_preservation(self): - self.assertEquals(type(json.loads(u'""')), unicode) - self.assertEquals(type(json.loads(u'"a"')), unicode) - self.assertEquals(type(json.loads(u'["a"]')[0]), unicode) - - def test_ensure_ascii_false_returns_unicode(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - self.assertEquals(type(json.dumps([], ensure_ascii=False)), unicode) - self.assertEquals(type(json.dumps(0, ensure_ascii=False)), unicode) - self.assertEquals(type(json.dumps({}, ensure_ascii=False)), unicode) - self.assertEquals(type(json.dumps("", ensure_ascii=False)), unicode) - - def test_ensure_ascii_false_bytestring_encoding(self): - # http://code.google.com/p/simplejson/issues/detail?id=48 - doc1 = {u'quux': 'Arr\xc3\xaat sur images'} - doc2 = {u'quux': u'Arr\xeat sur images'} - doc_ascii = '{"quux": "Arr\\u00eat sur images"}' - doc_unicode = u'{"quux": "Arr\xeat sur images"}' - self.assertEquals(json.dumps(doc1), doc_ascii) - self.assertEquals(json.dumps(doc2), doc_ascii) - self.assertEquals(json.dumps(doc1, ensure_ascii=False), doc_unicode) - self.assertEquals(json.dumps(doc2, ensure_ascii=False), doc_unicode) diff --git a/tablib/packages/simplejson/tool.py b/tablib/packages/simplejson/tool.py deleted file mode 100644 index 73370db..0000000 --- a/tablib/packages/simplejson/tool.py +++ /dev/null @@ -1,39 +0,0 @@ -r"""Command-line tool to validate and pretty-print JSON - -Usage:: - - $ echo '{"json":"obj"}' | python -m simplejson.tool - { - "json": "obj" - } - $ echo '{ 1.2:3.4}' | python -m simplejson.tool - Expecting property name: line 1 column 2 (char 2) - -""" -import sys -import simplejson as json - -def main(): - if len(sys.argv) == 1: - infile = sys.stdin - outfile = sys.stdout - elif len(sys.argv) == 2: - infile = open(sys.argv[1], 'rb') - outfile = sys.stdout - elif len(sys.argv) == 3: - infile = open(sys.argv[1], 'rb') - outfile = open(sys.argv[2], 'wb') - else: - raise SystemExit(sys.argv[0] + " [infile [outfile]]") - try: - obj = json.load(infile, - object_pairs_hook=json.OrderedDict, - use_decimal=True) - except ValueError, e: - raise SystemExit(e) - json.dump(obj, outfile, sort_keys=True, indent=' ', use_decimal=True) - outfile.write('\n') - - -if __name__ == '__main__': - main() diff --git a/test_suite.sh b/test_suite.sh index 157f457..5a1e7be 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -1,4 +1,5 @@ -nosetests test_tablib.py --with-xunit --with-coverage -coverage xml +rm -fr nosetests.xml +tox +# coverage xml rm -fr pylint.txt -pylint -d W0312 -d W0212 -d E1101 -d E0202 -d W0102 -d E0102 -f parseable ./tablib > pylint.txt || true
\ No newline at end of file +# pylint -d W0312 -d W0212 -d E1101 -d E0202 -d W0102 -d E0102 -f parseable ./tablib > pylint.txt || true
\ No newline at end of file diff --git a/test_tablib.py b/test_tablib.py index 15630f2..e4a4d11 100755 --- a/test_tablib.py +++ b/test_tablib.py @@ -12,452 +12,452 @@ import tablib class TablibTestCase(unittest.TestCase): - """Tablib test cases.""" + """Tablib test cases.""" - def setUp(self): - """Create simple data set with headers.""" + def setUp(self): + """Create simple data set with headers.""" - global data, book + global data, book - data = tablib.Dataset() - book = tablib.Databook() + data = tablib.Dataset() + book = tablib.Databook() - self.headers = ('first_name', 'last_name', 'gpa') - self.john = ('John', 'Adams', 90) - self.george = ('George', 'Washington', 67) - self.tom = ('Thomas', 'Jefferson', 50) + self.headers = ('first_name', 'last_name', 'gpa') + self.john = ('John', 'Adams', 90) + self.george = ('George', 'Washington', 67) + self.tom = ('Thomas', 'Jefferson', 50) - self.founders = tablib.Dataset(headers=self.headers) - self.founders.append(self.john) - self.founders.append(self.george) - self.founders.append(self.tom) + self.founders = tablib.Dataset(headers=self.headers) + self.founders.append(self.john) + self.founders.append(self.george) + self.founders.append(self.tom) - def tearDown(self): - """Teardown.""" - pass + def tearDown(self): + """Teardown.""" + pass - def test_empty_append(self): - """Verify append() correctly adds tuple with no headers.""" - new_row = (1, 2, 3) - data.append(new_row) + def test_empty_append(self): + """Verify append() correctly adds tuple with no headers.""" + new_row = (1, 2, 3) + data.append(new_row) - # Verify width/data - self.assertTrue(data.width == len(new_row)) - self.assertTrue(data[0] == new_row) + # Verify width/data + self.assertTrue(data.width == len(new_row)) + self.assertTrue(data[0] == new_row) - def test_empty_append_with_headers(self): - """Verify append() correctly detects mismatch of number of - headers and data. - """ - data.headers = ['first', 'second'] - new_row = (1, 2, 3, 4) + def test_empty_append_with_headers(self): + """Verify append() correctly detects mismatch of number of + headers and data. + """ + data.headers = ['first', 'second'] + new_row = (1, 2, 3, 4) - self.assertRaises(tablib.InvalidDimensions, data.append, new_row) + self.assertRaises(tablib.InvalidDimensions, data.append, new_row) - def test_add_column(self): - """Verify adding column works with/without headers.""" + def test_add_column(self): + """Verify adding column works with/without headers.""" - data.append(['kenneth']) - data.append(['bessie']) + data.append(['kenneth']) + data.append(['bessie']) - new_col = ['reitz', 'monke'] + new_col = ['reitz', 'monke'] - data.append(col=new_col) + data.append(col=new_col) - self.assertEquals(data[0], ('kenneth', 'reitz')) - self.assertEquals(data.width, 2) + self.assertEquals(data[0], ('kenneth', 'reitz')) + self.assertEquals(data.width, 2) - # With Headers - data.headers = ('fname', 'lname') - new_col = [21, 22] - data.append(col=new_col, header='age') + # With Headers + data.headers = ('fname', 'lname') + new_col = [21, 22] + data.append(col=new_col, header='age') - self.assertEquals(data['age'], new_col) + self.assertEquals(data['age'], new_col) - def test_add_column_no_data_no_headers(self): - """Verify adding new column with no headers.""" + def test_add_column_no_data_no_headers(self): + """Verify adding new column with no headers.""" - new_col = ('reitz', 'monke') + new_col = ('reitz', 'monke') - data.append(col=new_col) + data.append(col=new_col) - self.assertEquals(data[0], tuple([new_col[0]])) - self.assertEquals(data.width, 1) - self.assertEquals(data.height, len(new_col)) + self.assertEquals(data[0], tuple([new_col[0]])) + self.assertEquals(data.width, 1) + self.assertEquals(data.height, len(new_col)) - def test_add_callable_column(self): - """Verify adding column with values specified as callable.""" - new_col = [lambda x: x[0]] - self.founders.append(col=new_col, header='first_again') + def test_add_callable_column(self): + """Verify adding column with values specified as callable.""" + new_col = [lambda x: x[0]] + self.founders.append(col=new_col, header='first_again') # -# self.assertTrue(map(lambda x: x[0] == x[-1], self.founders)) +# self.assertTrue(map(lambda x: x[0] == x[-1], self.founders)) - def test_header_slicing(self): - """Verify slicing by headers.""" + def test_header_slicing(self): + """Verify slicing by headers.""" - self.assertEqual(self.founders['first_name'], - [self.john[0], self.george[0], self.tom[0]]) - self.assertEqual(self.founders['last_name'], - [self.john[1], self.george[1], self.tom[1]]) - self.assertEqual(self.founders['gpa'], - [self.john[2], self.george[2], self.tom[2]]) + self.assertEqual(self.founders['first_name'], + [self.john[0], self.george[0], self.tom[0]]) + self.assertEqual(self.founders['last_name'], + [self.john[1], self.george[1], self.tom[1]]) + self.assertEqual(self.founders['gpa'], + [self.john[2], self.george[2], self.tom[2]]) - def test_data_slicing(self): - """Verify slicing by data.""" + def test_data_slicing(self): + """Verify slicing by data.""" - # Slice individual rows - self.assertEqual(self.founders[0], self.john) - self.assertEqual(self.founders[:1], [self.john]) - self.assertEqual(self.founders[1:2], [self.george]) - self.assertEqual(self.founders[-1], self.tom) - self.assertEqual(self.founders[3:], []) + # Slice individual rows + self.assertEqual(self.founders[0], self.john) + self.assertEqual(self.founders[:1], [self.john]) + self.assertEqual(self.founders[1:2], [self.george]) + self.assertEqual(self.founders[-1], self.tom) + self.assertEqual(self.founders[3:], []) - # Slice multiple rows - self.assertEqual(self.founders[:], [self.john, self.george, self.tom]) - self.assertEqual(self.founders[0:2], [self.john, self.george]) - self.assertEqual(self.founders[1:3], [self.george, self.tom]) - self.assertEqual(self.founders[2:], [self.tom]) + # Slice multiple rows + self.assertEqual(self.founders[:], [self.john, self.george, self.tom]) + self.assertEqual(self.founders[0:2], [self.john, self.george]) + self.assertEqual(self.founders[1:3], [self.george, self.tom]) + self.assertEqual(self.founders[2:], [self.tom]) - def test_delete(self): - """Verify deleting from dataset works.""" + def test_delete(self): + """Verify deleting from dataset works.""" - # Delete from front of object - del self.founders[0] - self.assertEqual(self.founders[:], [self.george, self.tom]) + # Delete from front of object + del self.founders[0] + self.assertEqual(self.founders[:], [self.george, self.tom]) - # Verify dimensions, width should NOT change - self.assertEqual(self.founders.height, 2) - self.assertEqual(self.founders.width, 3) + # Verify dimensions, width should NOT change + self.assertEqual(self.founders.height, 2) + self.assertEqual(self.founders.width, 3) - # Delete from back of object - del self.founders[1] - self.assertEqual(self.founders[:], [self.george]) + # Delete from back of object + del self.founders[1] + self.assertEqual(self.founders[:], [self.george]) - # Verify dimensions, width should NOT change - self.assertEqual(self.founders.height, 1) - self.assertEqual(self.founders.width, 3) + # Verify dimensions, width should NOT change + self.assertEqual(self.founders.height, 1) + self.assertEqual(self.founders.width, 3) - # Delete from invalid index - self.assertRaises(IndexError, self.founders.__delitem__, 3) + # Delete from invalid index + self.assertRaises(IndexError, self.founders.__delitem__, 3) - def test_csv_export(self): - """Verify exporting dataset object as CSV.""" + def test_csv_export(self): + """Verify exporting dataset object as CSV.""" - # Build up the csv string with headers first, followed by each row - csv = '' - for col in self.headers: - csv += col + ',' + # Build up the csv string with headers first, followed by each row + csv = '' + for col in self.headers: + csv += col + ',' - csv = csv.strip(',') + '\r\n' + csv = csv.strip(',') + '\r\n' - for founder in self.founders: - for col in founder: - csv += str(col) + ',' - csv = csv.strip(',') + '\r\n' + for founder in self.founders: + for col in founder: + csv += str(col) + ',' + csv = csv.strip(',') + '\r\n' - self.assertEqual(csv, self.founders.csv) + self.assertEqual(csv, self.founders.csv) - def test_tsv_export(self): - """Verify exporting dataset object as CSV.""" + def test_tsv_export(self): + """Verify exporting dataset object as CSV.""" - # Build up the csv string with headers first, followed by each row - tsv = '' - for col in self.headers: - tsv += col + '\t' + # Build up the csv string with headers first, followed by each row + tsv = '' + for col in self.headers: + tsv += col + '\t' - tsv = tsv.strip('\t') + '\r\n' + tsv = tsv.strip('\t') + '\r\n' - for founder in self.founders: - for col in founder: - tsv += str(col) + '\t' - tsv = tsv.strip('\t') + '\r\n' + for founder in self.founders: + for col in founder: + tsv += str(col) + '\t' + tsv = tsv.strip('\t') + '\r\n' - self.assertEqual(tsv, self.founders.tsv) + self.assertEqual(tsv, self.founders.tsv) - def test_html_export(self): + def test_html_export(self): - """HTML export""" + """HTML export""" - html = markup.page() - html.table.open() - html.thead.open() + html = markup.page() + html.table.open() + html.thead.open() - html.tr(markup.oneliner.th(self.founders.headers)) - html.thead.close() + html.tr(markup.oneliner.th(self.founders.headers)) + html.thead.close() - for founder in self.founders: + for founder in self.founders: - html.tr(markup.oneliner.td(founder)) + html.tr(markup.oneliner.td(founder)) - html.table.close() - html = str(html) + html.table.close() + html = str(html) - self.assertEqual(html, self.founders.html) + self.assertEqual(html, self.founders.html) - def test_unicode_append(self): - """Passes in a single unicode charecter and exports.""" + def test_unicode_append(self): + """Passes in a single unicode charecter and exports.""" - new_row = ('å', 'é') - data.append(new_row) + new_row = ('å', 'é') + data.append(new_row) - data.json - data.yaml - data.csv - data.tsv - data.xls + data.json + data.yaml + data.csv + data.tsv + data.xls - def test_book_export_no_exceptions(self): - """Test that varoius exports don't error out.""" + def test_book_export_no_exceptions(self): + """Test that varoius exports don't error out.""" - book = tablib.Databook() - book.add_sheet(data) + book = tablib.Databook() + book.add_sheet(data) - book.json - book.yaml - book.xls + book.json + book.yaml + book.xls - def test_json_import_set(self): - """Generate and import JSON set serialization.""" - data.append(self.john) - data.append(self.george) - data.headers = self.headers + def test_json_import_set(self): + """Generate and import JSON set serialization.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers - _json = data.json + _json = data.json - data.json = _json + data.json = _json - self.assertEqual(_json, data.json) + self.assertEqual(_json, data.json) - def test_json_import_book(self): - """Generate and import JSON book serialization.""" - data.append(self.john) - data.append(self.george) - data.headers = self.headers + def test_json_import_book(self): + """Generate and import JSON book serialization.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers - book.add_sheet(data) - _json = book.json + book.add_sheet(data) + _json = book.json - book.json = _json + book.json = _json - self.assertEqual(_json, book.json) + self.assertEqual(_json, book.json) - def test_yaml_import_set(self): - """Generate and import YAML set serialization.""" - data.append(self.john) - data.append(self.george) - data.headers = self.headers + def test_yaml_import_set(self): + """Generate and import YAML set serialization.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers - _yaml = data.yaml + _yaml = data.yaml - data.yaml = _yaml + data.yaml = _yaml - self.assertEqual(_yaml, data.yaml) + self.assertEqual(_yaml, data.yaml) - def test_yaml_import_book(self): - """Generate and import YAML book serialization.""" - data.append(self.john) - data.append(self.george) - data.headers = self.headers + def test_yaml_import_book(self): + """Generate and import YAML book serialization.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers - book.add_sheet(data) - _yaml = book.yaml + book.add_sheet(data) + _yaml = book.yaml - book.yaml = _yaml + book.yaml = _yaml - self.assertEqual(_yaml, book.yaml) + self.assertEqual(_yaml, book.yaml) - def test_csv_import_set(self): - """Generate and import CSV set serialization.""" - data.append(self.john) - data.append(self.george) - data.headers = self.headers + def test_csv_import_set(self): + """Generate and import CSV set serialization.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers - _csv = data.csv + _csv = data.csv - data.csv = _csv + data.csv = _csv - self.assertEqual(_csv, data.csv) + self.assertEqual(_csv, data.csv) - def test_tsv_import_set(self): - """Generate and import TSV set serialization.""" - data.append(self.john) - data.append(self.george) - data.headers = self.headers + def test_tsv_import_set(self): + """Generate and import TSV set serialization.""" + data.append(self.john) + data.append(self.george) + data.headers = self.headers - _tsv = data.tsv + _tsv = data.tsv - data.tsv = _tsv + data.tsv = _tsv - self.assertEqual(_tsv, data.tsv) + self.assertEqual(_tsv, data.tsv) - def test_csv_format_detect(self): - """Test CSV format detection.""" + def test_csv_format_detect(self): + """Test CSV format detection.""" - _csv = ( - '1,2,3\n' - '4,5,6\n' - '7,8,9\n' - ) - _bunk = ( - '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' - ) + _csv = ( + '1,2,3\n' + '4,5,6\n' + '7,8,9\n' + ) + _bunk = ( + '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' + ) - self.assertTrue(tablib.formats.csv.detect(_csv)) - self.assertFalse(tablib.formats.csv.detect(_bunk)) + self.assertTrue(tablib.formats.csv.detect(_csv)) + self.assertFalse(tablib.formats.csv.detect(_bunk)) - def test_tsv_format_detect(self): - """Test TSV format detection.""" + def test_tsv_format_detect(self): + """Test TSV format detection.""" - _tsv = ( - '1\t2\t3\n' - '4\t5\t6\n' - '7\t8\t9\n' - ) - _bunk = ( - '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' - ) + _tsv = ( + '1\t2\t3\n' + '4\t5\t6\n' + '7\t8\t9\n' + ) + _bunk = ( + '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' + ) - self.assertTrue(tablib.formats.tsv.detect(_tsv)) - self.assertFalse(tablib.formats.tsv.detect(_bunk)) + self.assertTrue(tablib.formats.tsv.detect(_tsv)) + self.assertFalse(tablib.formats.tsv.detect(_bunk)) - def test_json_format_detect(self): - """Test JSON format detection.""" + def test_json_format_detect(self): + """Test JSON format detection.""" - _json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]' - _bunk = ( - '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' - ) + _json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]' + _bunk = ( + '¡¡¡¡¡¡¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' + ) - self.assertTrue(tablib.formats.json.detect(_json)) - self.assertFalse(tablib.formats.json.detect(_bunk)) + self.assertTrue(tablib.formats.json.detect(_json)) + self.assertFalse(tablib.formats.json.detect(_bunk)) - def test_yaml_format_detect(self): - """Test YAML format detection.""" + def test_yaml_format_detect(self): + """Test YAML format detection.""" - _yaml = '- {age: 90, first_name: John, last_name: Adams}' - _bunk = ( - '¡¡¡¡¡¡---///\n\n\n¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' - ) + _yaml = '- {age: 90, first_name: John, last_name: Adams}' + _bunk = ( + '¡¡¡¡¡¡---///\n\n\n¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' + ) - self.assertTrue(tablib.formats.yaml.detect(_yaml)) - self.assertFalse(tablib.formats.yaml.detect(_bunk)) + self.assertTrue(tablib.formats.yaml.detect(_yaml)) + self.assertFalse(tablib.formats.yaml.detect(_bunk)) - def test_auto_format_detect(self): - """Test auto format detection.""" + def test_auto_format_detect(self): + """Test auto format detection.""" - _yaml = '- {age: 90, first_name: John, last_name: Adams}' - _json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]' - _csv = '1,2,3\n4,5,6\n7,8,9\n' - _bunk = '¡¡¡¡¡¡---///\n\n\n¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' + _yaml = '- {age: 90, first_name: John, last_name: Adams}' + _json = '[{"last_name": "Adams","age": 90,"first_name": "John"}]' + _csv = '1,2,3\n4,5,6\n7,8,9\n' + _bunk = '¡¡¡¡¡¡---///\n\n\n¡¡£™∞¢£§∞§¶•¶ª∞¶•ªº••ª–º§•†•§º¶•†¥ª–º•§ƒø¥¨©πƒø†ˆ¥ç©¨√øˆ¥≈†ƒ¥ç©ø¨çˆ¥ƒçø¶' - self.assertEqual(tablib.detect(_yaml)[0], tablib.formats.yaml) - self.assertEqual(tablib.detect(_csv)[0], tablib.formats.csv) - self.assertEqual(tablib.detect(_json)[0], tablib.formats.json) - self.assertEqual(tablib.detect(_bunk)[0], None) + self.assertEqual(tablib.detect(_yaml)[0], tablib.formats.yaml) + self.assertEqual(tablib.detect(_csv)[0], tablib.formats.csv) + self.assertEqual(tablib.detect(_json)[0], tablib.formats.json) + self.assertEqual(tablib.detect(_bunk)[0], None) - def test_transpose(self): - """Transpose a dataset.""" + def test_transpose(self): + """Transpose a dataset.""" - transposed_founders = self.founders.transpose() - first_row = transposed_founders[0] - second_row = transposed_founders[1] + transposed_founders = self.founders.transpose() + first_row = transposed_founders[0] + second_row = transposed_founders[1] - self.assertEqual(transposed_founders.headers, - ["first_name","John", "George", "Thomas"]) - self.assertEqual(first_row, - ("last_name","Adams", "Washington", "Jefferson")) - self.assertEqual(second_row, - ("gpa",90, 67, 50)) + self.assertEqual(transposed_founders.headers, + ["first_name","John", "George", "Thomas"]) + self.assertEqual(first_row, + ("last_name","Adams", "Washington", "Jefferson")) + self.assertEqual(second_row, + ("gpa",90, 67, 50)) - def test_row_stacking(self): + def test_row_stacking(self): - """Row stacking.""" + """Row stacking.""" - to_join = tablib.Dataset(headers=self.founders.headers) + to_join = tablib.Dataset(headers=self.founders.headers) - for row in self.founders: - to_join.append(row=row) + for row in self.founders: + to_join.append(row=row) - row_stacked = self.founders.stack_rows(to_join) + row_stacked = self.founders.stack_rows(to_join) - for column in row_stacked.headers: + for column in row_stacked.headers: - original_data = self.founders[column] - expected_data = original_data + original_data - self.assertEqual(row_stacked[column], expected_data) + original_data = self.founders[column] + expected_data = original_data + original_data + self.assertEqual(row_stacked[column], expected_data) - def test_column_stacking(self): + def test_column_stacking(self): - """Column stacking""" + """Column stacking""" - to_join = tablib.Dataset(headers=self.founders.headers) + to_join = tablib.Dataset(headers=self.founders.headers) - for row in self.founders: - to_join.append(row=row) + for row in self.founders: + to_join.append(row=row) - column_stacked = self.founders.stack_columns(to_join) + column_stacked = self.founders.stack_columns(to_join) - for index, row in enumerate(column_stacked): + for index, row in enumerate(column_stacked): - original_data = self.founders[index] - expected_data = original_data + original_data - self.assertEqual(row, expected_data) + original_data = self.founders[index] + expected_data = original_data + original_data + self.assertEqual(row, expected_data) - self.assertEqual(column_stacked[0], - ("John", "Adams", 90, "John", "Adams", 90)) + self.assertEqual(column_stacked[0], + ("John", "Adams", 90, "John", "Adams", 90)) - def test_sorting(self): + def test_sorting(self): - """Sort columns.""" + """Sort columns.""" - sorted_data = self.founders.sort(col="first_name") + sorted_data = self.founders.sort(col="first_name") - first_row = sorted_data[0] - second_row = sorted_data[2] - third_row = sorted_data[1] - expected_first = self.founders[1] - expected_second = self.founders[2] - expected_third = self.founders[0] + first_row = sorted_data[0] + second_row = sorted_data[2] + third_row = sorted_data[1] + expected_first = self.founders[1] + expected_second = self.founders[2] + expected_third = self.founders[0] - self.assertEqual(first_row, expected_first) - self.assertEqual(second_row, expected_second) - self.assertEqual(third_row, expected_third) + self.assertEqual(first_row, expected_first) + self.assertEqual(second_row, expected_second) + self.assertEqual(third_row, expected_third) - def test_wipe(self): - """Purge a dataset.""" + def test_wipe(self): + """Purge a dataset.""" - new_row = (1, 2, 3) - data.append(new_row) + new_row = (1, 2, 3) + data.append(new_row) - # Verify width/data - self.assertTrue(data.width == len(new_row)) - self.assertTrue(data[0] == new_row) + # Verify width/data + self.assertTrue(data.width == len(new_row)) + self.assertTrue(data[0] == new_row) - data.wipe() - new_row = (1, 2, 3, 4) - data.append(new_row) - self.assertTrue(data.width == len(new_row)) - self.assertTrue(data[0] == new_row) + data.wipe() + new_row = (1, 2, 3, 4) + data.append(new_row) + self.assertTrue(data.width == len(new_row)) + self.assertTrue(data[0] == new_row) if __name__ == '__main__': - unittest.main() + unittest.main() @@ -0,0 +1,9 @@ +[tox] +envlist = py24,py25,py26,py27 + +[testenv] +commands=py.test --junitxml=junit-{envname}.xml +deps = + nose + simplejson + pytest
\ No newline at end of file |