diff options
60 files changed, 1334 insertions, 700 deletions
@@ -6,3 +6,4 @@ docs/_build build dist .DS_Store +.tox diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d919621 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.3" +install: + - "python setup.py install" +script: python setup.py test @@ -29,3 +29,5 @@ Patches and suggestions: - Peter van Dijk (Habbie) - Stefan Ebner - Rene Leonhardt +- Thomas Waldmann +- Cory Benfield (Lukasa) @@ -13,6 +13,21 @@ Version 2.7 - Added `urlencode` filter that automatically quotes values for URL safe usage with utf-8 as only supported encoding. If applications want to change this encoding they can override the filter. +- Added `keep-trailing-newline` configuration to environments and + templates to optionally preserve the final trailing newline. +- Accessing `last` on the loop context no longer causes the iterator + to be consumed into a list. +- Python requirement changed: 2.6, 2.7 or >= 3.3 are required now, + supported by same source code, using the "six" compatibility library. +- Allow `contextfunction` and other decorators to be applied to `__call__`. +- Added support for changing from newline to different signs in the `wordwrap` + filter. +- Added support for ignoring memcache errors silently. +- Added support for keeping the trailing newline in templates. +- Added finer grained support for stripping whitespace on the left side + of blocks. +- Added `map`, `select`, `reject`, `selectattr` and `rejectattr` + filters. Version 2.6 ----------- diff --git a/MANIFEST.in b/MANIFEST.in index aeb66af..63760bb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include MANIFEST.in Makefile CHANGES LICENSE AUTHORS jinja2/_debugsupport.c +include MANIFEST.in Makefile CHANGES LICENSE AUTHORS recursive-include docs * recursive-include custom_fixers * recursive-include ext * @@ -1 +0,0 @@ -Basic Support for Jinja2 in Textmate diff --git a/custom_fixers/__init__.py b/custom_fixers/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/custom_fixers/__init__.py +++ /dev/null diff --git a/custom_fixers/fix_alt_unicode.py b/custom_fixers/fix_alt_unicode.py deleted file mode 100644 index 96a81c1..0000000 --- a/custom_fixers/fix_alt_unicode.py +++ /dev/null @@ -1,13 +0,0 @@ -from lib2to3 import fixer_base -from lib2to3.fixer_util import Name, BlankLine - - -class FixAltUnicode(fixer_base.BaseFix): - PATTERN = """ - func=funcdef< 'def' name='__unicode__' - parameters< '(' NAME ')' > any+ > - """ - - def transform(self, node, results): - name = results['name'] - name.replace(Name('__str__', prefix=name.prefix)) diff --git a/custom_fixers/fix_broken_reraising.py b/custom_fixers/fix_broken_reraising.py deleted file mode 100644 index fd0ea68..0000000 --- a/custom_fixers/fix_broken_reraising.py +++ /dev/null @@ -1,21 +0,0 @@ -from lib2to3 import fixer_base, pytree -from lib2to3.fixer_util import Name, BlankLine, Name, Attr, ArgList - - -class FixBrokenReraising(fixer_base.BaseFix): - PATTERN = """ - raise_stmt< 'raise' any ',' val=any ',' tb=any > - """ - - # run before the broken 2to3 checker with the same goal - # tries to rewrite it with a rule that does not work out for jinja - run_order = 1 - - def transform(self, node, results): - tb = results['tb'].clone() - tb.prefix = '' - with_tb = Attr(results['val'].clone(), Name('with_traceback')) + \ - [ArgList([tb])] - new = pytree.Node(self.syms.simple_stmt, [Name("raise")] + with_tb) - new.prefix = node.prefix - return new diff --git a/custom_fixers/fix_xrange2.py b/custom_fixers/fix_xrange2.py deleted file mode 100644 index 5d35e50..0000000 --- a/custom_fixers/fix_xrange2.py +++ /dev/null @@ -1,11 +0,0 @@ -from lib2to3 import fixer_base -from lib2to3.fixer_util import Name, BlankLine - - -# whyever this is necessary.. - -class FixXrange2(fixer_base.BaseFix): - PATTERN = "'xrange'" - - def transform(self, node, results): - node.replace(Name('range', prefix=node.prefix)) diff --git a/docs/extensions.rst b/docs/extensions.rst index 357f592..3878d8c 100644 --- a/docs/extensions.rst +++ b/docs/extensions.rst @@ -219,7 +219,7 @@ common use cases. The i18n extension is a good example of why extensions are useful, another one would be fragment caching. When writing extensions you have to keep in mind that you are working with the -Jinja2 template compiler which does not validate the node tree you are possing +Jinja2 template compiler which does not validate the node tree you are passing to it. If the AST is malformed you will get all kinds of compiler or runtime errors that are horrible to debug. Always make sure you are using the nodes you create correctly. The API documentation below shows which nodes exist and diff --git a/docs/faq.rst b/docs/faq.rst index d066bff..f66bd81 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -139,7 +139,7 @@ Jinja2 no longer ships with a C implementation of it but only the pure Python implementation. It will however check if MarkupSafe is available and installed, and if it is, use the Markup class from MarkupSafe. -So if you want the speedups, just import MarkupSafe. +So if you want the speedups, just install MarkupSafe. .. _MarkupSafe: http://pypi.python.org/pypi/MarkupSafe @@ -174,7 +174,7 @@ harder to maintain the code for older Python versions. If you really need Python 2.3 support you either have to use `Jinja 1`_ or other templating engines that still support 2.3. -My Macros are overriden by something +My Macros are overridden by something ------------------------------------ In some situations the Jinja scoping appears arbitrary: diff --git a/docs/integration.rst b/docs/integration.rst index e5c76dc..79dff76 100644 --- a/docs/integration.rst +++ b/docs/integration.rst @@ -5,6 +5,9 @@ Jinja2 provides some code for integration into other tools such as frameworks, the `Babel`_ library or your favourite editor for fancy code highlighting. This is a brief description of whats included. +Files to help integration are available +`here. <https://github.com/mitsuhiko/jinja2/tree/master/ext>`_ + .. _babel-integration: Babel Integration @@ -78,18 +81,18 @@ snippet and add it into your `config/environment.py`:: TextMate -------- -Inside the `ext` folder of Jinja2 there is a bundle for TextMate that supports -syntax highlighting for Jinja1 and Jinja2 for text based templates as well as -HTML. It also contains a few often used snippets. +Inside the `ext` folder at the root of the Jinja2 project there is a bundle for +TextMate that supports syntax highlighting for Jinja1 and Jinja2 for text based +templates as well as HTML. It also contains a few often used snippets. Vim --- A syntax plugin for `Vim`_ exists in the Vim-scripts directory as well as the -ext folder of Jinja2. `The script <http://www.vim.org/scripts/script.php?script_id=1856>`_ -supports Jinja1 and Jinja2. Once installed two file types are available `jinja` -and `htmljinja`. The first one for text based templates, the latter for HTML -templates. +`ext` folder at the root of the Jinja2 project. `The script +<http://www.vim.org/scripts/script.php?script_id=1856>`_ supports Jinja1 and +Jinja2. Once installed two file types are available `jinja` and `htmljinja`. +The first one for text based templates, the latter for HTML templates. Copy the files into your `syntax` folder. diff --git a/docs/intro.rst b/docs/intro.rst index b08947d..5d7e9f8 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -2,8 +2,7 @@ Introduction ============ This is the documentation for the Jinja2 general purpose templating language. -Jinja2 is a library for Python 2.4 and onwards that is designed to be flexible, -fast and secure. +Jinja2 is a library for Python that is designed to be flexible, fast and secure. If you have any exposure to other text-based template languages, such as Smarty or Django, you should feel right at home with Jinja2. It's both designer and @@ -13,15 +12,12 @@ useful for templating environments. Prerequisites ------------- -Jinja2 needs at least **Python 2.4** to run. Additionally a working C-compiler -that can create python extensions should be installed for the debugger if you -are using Python 2.4. - -If you don't have a working C-compiler and you are trying to install the source -release with the debugsupport you will get a compiler error. - -.. _ctypes: http://python.net/crew/theller/ctypes/ +Jinja2 works with Python 2.6.x, 2.7.x and >= 3.3. If you are using Python +3.2 you can use an older release of Jinja2 (2.6) as support for 3.2 was +dropped in 2.7. +If you wish to use the :class:`~jinja2.PackageLoader` class, you will also +need setuptools or distribute installed at runtime. Installation ------------ @@ -40,7 +36,7 @@ You can install the most recent Jinja2 version using `easy_install`_ or `pip`_:: This will install a Jinja2 egg in your Python installation's site-packages directory. -(If you are installing from the windows command line omit the `sudo` and make +(If you are installing from the Windows command line omit the `sudo` and make sure to run the command as user with administrator rights) From the tarball release @@ -50,7 +46,7 @@ From the tarball release 2. Unpack the tarball 3. ``sudo python setup.py install`` -Note that you either have to have setuptools or `distribute`_ installed, +Note that you either have to have setuptools or `distribute`_ installed; the latter is preferred. This will install Jinja2 into your Python installation's site-packages directory. @@ -93,12 +89,14 @@ using Jinja2 with autoescaping. .. _MarkupSafe: http://pypi.python.org/pypi/MarkupSafe +<<<<<<< HEAD +======= Enable the debug support Module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default Jinja2 will not compile the debug support module. Enabling this will fail if you don't have the Python headers or a working compiler. This -is often the case if you are installing Jinja2 from a windows machine. +is often the case if you are installing Jinja2 from a Windows machine. Because the debug support is only necessary for Python 2.4 you will not have to do this unless you run 2.4:: @@ -106,6 +104,7 @@ have to do this unless you run 2.4:: sudo python setup.py --with-debugsupport install +>>>>>>> Talksum/distribute_required Basic API Usage --------------- @@ -136,7 +135,7 @@ indeed using unicode internally. Experimental Python 3 Support ----------------------------- -Jinja 2.3 brings experimental support for Python 3. It means that all +Jinja 2.7 brings experimental support for Python >=3.3. It means that all unittests pass on the new version, but there might still be small bugs in there and behavior might be inconsistent. If you notice any bugs, please provide feedback in the `Jinja bug tracker`_. diff --git a/docs/jinjaext.py b/docs/jinjaext.py index 12b5447..8395a55 100644 --- a/docs/jinjaext.py +++ b/docs/jinjaext.py @@ -8,6 +8,7 @@ :copyright: Copyright 2008 by Armin Ronacher. :license: BSD. """ +import collections import os import re import inspect @@ -22,6 +23,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ Number, Operator, Generic from jinja2 import Environment, FileSystemLoader +from jinja2.utils import next def parse_rst(state, content_offset, doc): @@ -110,10 +112,10 @@ def dump_functions(mapping): def directive(dirname, arguments, options, content, lineno, content_offset, block_text, state, state_machine): reverse_mapping = {} - for name, func in mapping.iteritems(): + for name, func in mapping.items(): reverse_mapping.setdefault(func, []).append(name) filters = [] - for func, names in reverse_mapping.iteritems(): + for func, names in reverse_mapping.items(): aliases = sorted(names, key=lambda x: len(x)) name = aliases.pop() filters.append((name, aliases, func)) @@ -145,9 +147,9 @@ def jinja_nodes(dirname, arguments, options, content, lineno, doc.append(p + '.. autoclass:: %s(%s)' % (node.__name__, sig), '') if node.abstract: members = [] - for key, name in node.__dict__.iteritems(): + for key, name in node.__dict__.items(): if not key.startswith('_') and \ - not hasattr(node.__base__, key) and callable(name): + not hasattr(node.__base__, key) and isinstance(name, collections.Callable): members.append(key) if members: members.sort() @@ -169,10 +171,10 @@ def inject_toc(app, doctree, docname): titleiter = iter(doctree.traverse(nodes.title)) try: # skip first title, we are not interested in that one - titleiter.next() - title = titleiter.next() + next(titleiter) + title = next(titleiter) # and check if there is at least another title - titleiter.next() + next(titleiter) except StopIteration: return tocnode = nodes.section('') diff --git a/docs/sandbox.rst b/docs/sandbox.rst index 1302f18..33de4d2 100644 --- a/docs/sandbox.rst +++ b/docs/sandbox.rst @@ -42,7 +42,7 @@ API are running multiple users on the same server) they can't harm each other via JavaScript insertions and much more. - Also the sandbox is only as good as the configuration. We stronly + Also the sandbox is only as good as the configuration. We strongly recommend only passing non-shared resources to the template and use some sort of whitelisting for attributes. diff --git a/docs/switching.rst b/docs/switching.rst index ba3cfb1..68cdb33 100644 --- a/docs/switching.rst +++ b/docs/switching.rst @@ -3,7 +3,7 @@ Switching from other Template Engines .. highlight:: html+jinja -If you have used a different template engine in the past and want to swtich +If you have used a different template engine in the past and want to switch to Jinja2 here is a small guide that shows the basic syntatic and semantic changes between some common, similar text template engines for Python. @@ -177,9 +177,25 @@ operator. Here are some examples:: Loops ~~~~~ -For loops work very similar to Django, the only incompatibility is that in -Jinja2 the special variable for the loop context is called `loop` and not -`forloop` like in Django. +For loops work very similar to Django. Notably, in Jinja2 the special variable for +the loop context is called `loop` and not `forloop` like in Django. + +In addition, the Django `empty` argument is called `else` in Jinja2. For example, the +Django template:: + + {% for item in items %} + {{ item }} + {% empty %} + No items! + {% endfor %} + +would be handled in Jinja2 as:: + + {% for item in items %} + {{ item }} + {% else %} + No items! + {% endfor %} Cycle ~~~~~ @@ -213,9 +229,9 @@ Jinja2 to look more like Mako: .. sourcecode:: python - env = Environment('<%', '%>', '${', '}', '%') + env = Environment('<%', '%>', '${', '}', '<%doc>', '</%doc>', '%', '##') -Once the environment is configure like that Jinja2 should be able to interpret +Once the environment is configured like that Jinja2 should be able to interpret a small subset of Mako templates. Jinja2 does not support embedded Python code so you would have to move that out of the template. The syntax for defs (in Jinja2 defs are called macros) and template inheritance is different too. The diff --git a/docs/templates.rst b/docs/templates.rst index 3032e3a..6bc6e04 100644 --- a/docs/templates.rst +++ b/docs/templates.rst @@ -152,12 +152,49 @@ yourself:: Whitespace Control ------------------ -In the default configuration whitespace is not further modified by the -template engine, so each whitespace (spaces, tabs, newlines etc.) is returned -unchanged. If the application configures Jinja to `trim_blocks` the first -newline after a a template tag is removed automatically (like in PHP). +In the default configuration, a single trailing newline is stripped if +present, and whitespace is not further modified by the template engine. Each +whitespace (spaces, tabs, newlines etc.) is returned unchanged. If the +application configures Jinja to `trim_blocks` the first newline after a +template tag is removed automatically (like in PHP). The `lstrip_blocks` +option can also be set to strip tabs and spaces from the beginning of +line to the start of a block. (Nothing will be stripped if there are +other characters before the start of the block.) + +With both `trim_blocks` and `lstrip_blocks` enabled you can put block tags +on their own lines, and the entire block line will be removed when +rendered, preserving the whitespace of the contents. For example, +without the `trim_blocks` and `lstrip_blocks` options, this template:: + + <div> + {% if True %} + yay + {% endif %} + </div> + +gets rendered with blank lines inside the div:: + + <div> + + yay + + </div> + +But with both `trim_blocks` and `lstrip_blocks` enabled, the lines with the +template blocks are removed while preserving the whitespace of the contents:: + + <div> + yay + </div> -But you can also strip whitespace in templates by hand. If you put an minus +You can manually disable the `lstrip_blocks` behavior by putting a +plus sign (``+``) at the start of a block:: + + <div> + {%+ if something %}yay{% endif %} + </div> + +You can also strip whitespace in templates by hand. If you put an minus sign (``-``) to the start or end of an block (for example a for tag), a comment or variable expression you can remove the whitespaces after or before that block:: @@ -172,6 +209,10 @@ a list of numbers from ``1`` to ``9`` the output would be ``123456789``. If :ref:`line-statements` are enabled they strip leading whitespace automatically up to the beginning of the line. +Jinja2 by default also removes trailing newlines. To keep the single +trailing newline when it is present, configure Jinja to +`keep_trailing_newline`. + .. admonition:: Note You must not use a whitespace between the tag and the minus sign. @@ -332,7 +373,7 @@ advantage of it, see :ref:`null-master-fallback`. The filename of the template depends on the template loader. For example the :class:`FileSystemLoader` allows you to access other templates by giving the -filename. You can access templates in subdirectories with an slash:: +filename. You can access templates in subdirectories with a slash:: {% extends "layout/default.html" %} @@ -571,6 +612,10 @@ by using `else`:: {% endfor %} </ul> +Note that in Python `else` blocks are executed whenever the corresponding +loop did not `break`. Since in Jinja loops cannot `break` anyway, +a slightly different behavior of the `else` keyword was chosen. + It is also possible to use loops recursively. This is useful if you are dealing with recursive data such as sitemaps. To use loops recursively you basically have to add the `recursive` modifier to the loop definition and @@ -587,6 +632,10 @@ The following example implements a sitemap with recursive loops:: {%- endfor %} </ul> +The `loop` variable always refers to the closest (innermost) loop. If we +have more than one levels of loops, we can rebind the variable `loop` by +writing `{% set outer_loop = loop %}` after the loop that we want to +use recursively. Then, we can call it using `{{ outer_loop(...) }}` If ~~ @@ -785,7 +834,7 @@ default. For more details about context behavior of imports and includes see :ref:`import-visibility`. From Jinja 2.2 onwards you can mark an include with ``ignore missing`` in -which case Jinja will ignore the statement if the template to be ignored +which case Jinja will ignore the statement if the template to be included does not exist. When combined with ``with`` or ``without context`` it has to be placed *before* the context visibility statement. Here some valid examples:: @@ -819,7 +868,7 @@ Jinja2 supports putting often used code into macros. These macros can go into different templates and get imported from there. This works similar to the import statements in Python. It's important to know that imports are cached and imported templates don't have access to the current template variables, -just the globals by defualt. For more details about context behavior of +just the globals by default. For more details about context behavior of imports and includes see :ref:`import-visibility`. There are two ways to import templates. You can import the complete template @@ -861,7 +910,7 @@ namespace:: </dl> <p>{{ textarea('comment') }}</p> -Macros and variables starting with one ore more underscores are private and +Macros and variables starting with one or more underscores are private and cannot be imported. .. versionchanged:: 2.4 diff --git a/docs/tricks.rst b/docs/tricks.rst index 566575e..5427a3e 100644 --- a/docs/tricks.rst +++ b/docs/tricks.rst @@ -75,7 +75,7 @@ sense to defined a default for that variable:: <ul id="navigation"> {% for href, id, caption in navigation_bar %} <li{% if id == active_page %} class="active"{% endif - %}><a href="{{ href|e }}">{{ caption|e }}</a>/li> + %}><a href="{{ href|e }}">{{ caption|e }}</a></li> {% endfor %} </ul> ... diff --git a/ext/Vim/jinja.vim b/ext/Vim/jinja.vim index 894dcc4..503aec6 100644 --- a/ext/Vim/jinja.vim +++ b/ext/Vim/jinja.vim @@ -49,14 +49,14 @@ syn keyword jinjaStatement containedin=jinjaTagBlock contained macro skipwhite n syn keyword jinjaStatement containedin=jinjaTagBlock contained block skipwhite nextgroup=jinjaBlockName " Variable Names -syn match jinjaVariable containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ +syn match jinjaVariable containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained /[a-zA-Z_][a-zA-Z0-9_]*/ syn keyword jinjaSpecial containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained false true none False True None loop super caller varargs kwargs " Filters -syn match jinjaOperator "|" containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained nextgroup=jinjaFilter -syn match jinjaFilter contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ -syn match jinjaFunction contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ -syn match jinjaBlockName contained skipwhite /[a-zA-Z_][a-zA-Z0-9_]*/ +syn match jinjaOperator "|" containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained skipwhite nextgroup=jinjaFilter +syn match jinjaFilter contained /[a-zA-Z_][a-zA-Z0-9_]*/ +syn match jinjaFunction contained /[a-zA-Z_][a-zA-Z0-9_]*/ +syn match jinjaBlockName contained /[a-zA-Z_][a-zA-Z0-9_]*/ " Jinja template constants syn region jinjaString containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained start=/"/ skip=/\\"/ end=/"/ @@ -73,7 +73,7 @@ syn match jinjaAttribute contained /[a-zA-Z_][a-zA-Z0-9_]*/ syn region jinjaNested matchgroup=jinjaOperator start="(" end=")" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained syn region jinjaNested matchgroup=jinjaOperator start="\[" end="\]" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained syn region jinjaNested matchgroup=jinjaOperator start="{" end="}" transparent display containedin=jinjaVarBlock,jinjaTagBlock,jinjaNested contained -syn region jinjaTagBlock matchgroup=jinjaTagDelim start=/{%-\?/ end=/-\?%}/ skipwhite containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment +syn region jinjaTagBlock matchgroup=jinjaTagDelim start=/{%-\?/ end=/-\?%}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment syn region jinjaVarBlock matchgroup=jinjaVarDelim start=/{{-\?/ end=/-\?}}/ containedin=ALLBUT,jinjaTagBlock,jinjaVarBlock,jinjaRaw,jinjaString,jinjaNested,jinjaComment @@ -86,10 +86,10 @@ syn region jinjaComment matchgroup=jinjaCommentDelim start="{#" end="#}" contain " Block start keywords. A bit tricker. We only highlight at the start of a " tag block and only if the name is not followed by a comma or equals sign " which usually means that we have to deal with an assignment. -syn match jinjaStatement containedin=jinjaTagBlock contained skipwhite /\({%-\?\s*\)\@<=\<[a-zA-Z_][a-zA-Z0-9_]*\>\(\s*[,=]\)\@!/ +syn match jinjaStatement containedin=jinjaTagBlock contained /\({%-\?\s*\)\@<=\<[a-zA-Z_][a-zA-Z0-9_]*\>\(\s*[,=]\)\@!/ " and context modifiers -syn match jinjaStatement containedin=jinjaTagBlock contained /\<with\(out\)\?\s\+context\>/ skipwhite +syn match jinjaStatement containedin=jinjaTagBlock contained /\<with\(out\)\?\s\+context\>/ " Define the default highlighting. diff --git a/ext/django2jinja/django2jinja.py b/ext/django2jinja/django2jinja.py index 6d9e76c..ad9627f 100644 --- a/ext/django2jinja/django2jinja.py +++ b/ext/django2jinja/django2jinja.py @@ -609,7 +609,7 @@ def width_ratio(writer, node): @node(core_tags.WithNode) def with_block(writer, node): writer.warn('with block expanded into set statement. This could cause ' - 'variables following that block to be overriden.', node) + 'variables following that block to be overridden.', node) writer.start_block() writer.write('set %s = ' % node.name) writer.node(node.var) diff --git a/jinja2/_compat.py b/jinja2/_compat.py new file mode 100644 index 0000000..6318f0b --- /dev/null +++ b/jinja2/_compat.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +""" + jinja2._compat + ~~~~~~~~~~~~~~ + + Some py2/py3 compatibility support that is not yet available in + "six" 1.3.0. + There are bugs open for "six" for all this stuff, so we can remove it + again from here as soon as we require a new enough "six" release. + + :copyright: Copyright 2013 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# https://bitbucket.org/gutworth/six/issue/25/add-unichr +try: + unichr = unichr # py2 +except NameError: + unichr = chr # py3 + +try: + range_type = xrange +except NameError: + range_type = range diff --git a/jinja2/_debugsupport.c b/jinja2/_debugsupport.c deleted file mode 100644 index e756d8e..0000000 --- a/jinja2/_debugsupport.c +++ /dev/null @@ -1,78 +0,0 @@ -/** - * jinja2._debugsupport - * ~~~~~~~~~~~~~~~~~~~~ - * - * C implementation of `tb_set_next`. - * - * :copyright: (c) 2010 by the Jinja Team. - * :license: BSD. - */ - -#include <Python.h> - - -static PyObject* -tb_set_next(PyObject *self, PyObject *args) -{ - PyTracebackObject *tb, *old; - PyObject *next; - - if (!PyArg_ParseTuple(args, "O!O:tb_set_next", &PyTraceBack_Type, &tb, &next)) - return NULL; - if (next == Py_None) - next = NULL; - else if (!PyTraceBack_Check(next)) { - PyErr_SetString(PyExc_TypeError, - "tb_set_next arg 2 must be traceback or None"); - return NULL; - } - else - Py_INCREF(next); - - old = tb->tb_next; - tb->tb_next = (PyTracebackObject*)next; - Py_XDECREF(old); - - Py_INCREF(Py_None); - return Py_None; -} - -static PyMethodDef module_methods[] = { - {"tb_set_next", (PyCFunction)tb_set_next, METH_VARARGS, - "Set the tb_next member of a traceback object."}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - - -#if PY_MAJOR_VERSION < 3 - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif -PyMODINIT_FUNC -init_debugsupport(void) -{ - Py_InitModule3("jinja2._debugsupport", module_methods, ""); -} - -#else /* Python 3.x module initialization */ - -static struct PyModuleDef module_definition = { - PyModuleDef_HEAD_INIT, - "jinja2._debugsupport", - NULL, - -1, - module_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit__debugsupport(void) -{ - return PyModule_Create(&module_definition); -} - -#endif diff --git a/jinja2/_markupsafe/__init__.py b/jinja2/_markupsafe/__init__.py index ec7bd57..5ffb57d 100644 --- a/jinja2/_markupsafe/__init__.py +++ b/jinja2/_markupsafe/__init__.py @@ -9,8 +9,9 @@ :license: BSD, see LICENSE for more details. """ import re -from itertools import imap - +import six +from six.moves import map +from jinja2._compat import unichr __all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] @@ -19,7 +20,7 @@ _striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)') _entity_re = re.compile(r'&([^;]+);') -class Markup(unicode): +class Markup(six.text_type): r"""Marks a string as being safe for inclusion in HTML/XML output without needing to be escaped. This implements the `__html__` interface a couple of frameworks and web applications use. :class:`Markup` is a direct @@ -68,56 +69,56 @@ class Markup(unicode): if hasattr(base, '__html__'): base = base.__html__() if encoding is None: - return unicode.__new__(cls, base) - return unicode.__new__(cls, base, encoding, errors) + return six.text_type.__new__(cls, base) + return six.text_type.__new__(cls, base, encoding, errors) def __html__(self): return self def __add__(self, other): - if hasattr(other, '__html__') or isinstance(other, basestring): - return self.__class__(unicode(self) + unicode(escape(other))) + if hasattr(other, '__html__') or isinstance(other, six.string_types): + return self.__class__(six.text_type(self) + six.text_type(escape(other))) return NotImplemented def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, basestring): - return self.__class__(unicode(escape(other)) + unicode(self)) + if hasattr(other, '__html__') or isinstance(other, six.string_types): + return self.__class__(six.text_type(escape(other)) + six.text_type(self)) return NotImplemented def __mul__(self, num): if isinstance(num, (int, long)): - return self.__class__(unicode.__mul__(self, num)) + return self.__class__(six.text_type.__mul__(self, num)) return NotImplemented __rmul__ = __mul__ def __mod__(self, arg): if isinstance(arg, tuple): - arg = tuple(imap(_MarkupEscapeHelper, arg)) + arg = tuple(map(_MarkupEscapeHelper, arg)) else: arg = _MarkupEscapeHelper(arg) - return self.__class__(unicode.__mod__(self, arg)) + return self.__class__(six.text_type.__mod__(self, arg)) def __repr__(self): return '%s(%s)' % ( self.__class__.__name__, - unicode.__repr__(self) + six.text_type.__repr__(self) ) def join(self, seq): - return self.__class__(unicode.join(self, imap(escape, seq))) - join.__doc__ = unicode.join.__doc__ + return self.__class__(six.text_type.join(self, map(escape, seq))) + join.__doc__ = six.text_type.join.__doc__ def split(self, *args, **kwargs): - return map(self.__class__, unicode.split(self, *args, **kwargs)) - split.__doc__ = unicode.split.__doc__ + return map(self.__class__, six.text_type.split(self, *args, **kwargs)) + split.__doc__ = six.text_type.split.__doc__ def rsplit(self, *args, **kwargs): - return map(self.__class__, unicode.rsplit(self, *args, **kwargs)) - rsplit.__doc__ = unicode.rsplit.__doc__ + return map(self.__class__, six.text_type.rsplit(self, *args, **kwargs)) + rsplit.__doc__ = six.text_type.rsplit.__doc__ def splitlines(self, *args, **kwargs): - return map(self.__class__, unicode.splitlines(self, *args, **kwargs)) - splitlines.__doc__ = unicode.splitlines.__doc__ + return map(self.__class__, six.text_type.splitlines(self, *args, **kwargs)) + splitlines.__doc__ = six.text_type.splitlines.__doc__ def unescape(self): r"""Unescape markup again into an unicode string. This also resolves @@ -139,7 +140,7 @@ class Markup(unicode): except ValueError: pass return u'' - return _entity_re.sub(handle_match, unicode(self)) + return _entity_re.sub(handle_match, six.text_type(self)) def striptags(self): r"""Unescape markup into an unicode string and strip all tags. This @@ -164,10 +165,10 @@ class Markup(unicode): return rv def make_wrapper(name): - orig = getattr(unicode, name) + orig = getattr(six.text_type, name) def func(self, *args, **kwargs): args = _escape_argspec(list(args), enumerate(args)) - _escape_argspec(kwargs, kwargs.iteritems()) + _escape_argspec(kwargs, six.iteritems(kwargs)) return self.__class__(orig(self, *args, **kwargs)) func.__name__ = orig.__name__ func.__doc__ = orig.__doc__ @@ -175,21 +176,13 @@ class Markup(unicode): for method in '__getitem__', 'capitalize', \ 'title', 'lower', 'upper', 'replace', 'ljust', \ + 'format', 'partition', 'rpartition', \ 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ 'translate', 'expandtabs', 'swapcase', 'zfill': locals()[method] = make_wrapper(method) - # new in python 2.5 - if hasattr(unicode, 'partition'): - partition = make_wrapper('partition'), - rpartition = make_wrapper('rpartition') - - # new in python 2.6 - if hasattr(unicode, 'format'): - format = make_wrapper('format') - # not in python 3 - if hasattr(unicode, '__getslice__'): + if hasattr(six.text_type, '__getslice__'): __getslice__ = make_wrapper('__getslice__') del method, make_wrapper @@ -198,7 +191,7 @@ class Markup(unicode): def _escape_argspec(obj, iterable): """Helper for various string-wrapped functions.""" for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, basestring): + if hasattr(value, '__html__') or isinstance(value, six.string_types): obj[key] = escape(value) return obj @@ -211,7 +204,7 @@ class _MarkupEscapeHelper(object): __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x]) __str__ = lambda s: str(escape(s.obj)) - __unicode__ = lambda s: unicode(escape(s.obj)) + __unicode__ = lambda s: six.text_type(escape(s.obj)) __repr__ = lambda s: str(escape(repr(s.obj))) __int__ = lambda s: int(s.obj) __float__ = lambda s: float(s.obj) diff --git a/jinja2/_markupsafe/_bundle.py b/jinja2/_markupsafe/_bundle.py index e694faf..d23730f 100644 --- a/jinja2/_markupsafe/_bundle.py +++ b/jinja2/_markupsafe/_bundle.py @@ -10,6 +10,7 @@ :copyright: Copyright 2010 by the Jinja team, see AUTHORS. :license: BSD, see LICENSE for details. """ +from __future__ import print_function import sys import os import re @@ -25,7 +26,7 @@ def rewrite_imports(lines): def main(): if len(sys.argv) != 2: - print 'error: only argument is path to markupsafe' + print('error: only argument is path to markupsafe') sys.exit(1) basedir = os.path.dirname(__file__) markupdir = sys.argv[1] diff --git a/jinja2/_markupsafe/_native.py b/jinja2/_markupsafe/_native.py index 7b95828..389be74 100644 --- a/jinja2/_markupsafe/_native.py +++ b/jinja2/_markupsafe/_native.py @@ -9,6 +9,7 @@ :license: BSD, see LICENSE for more details. """ from jinja2._markupsafe import Markup +import six def escape(s): @@ -18,7 +19,7 @@ def escape(s): """ if hasattr(s, '__html__'): return s.__html__() - return Markup(unicode(s) + return Markup(six.text_type(s) .replace('&', '&') .replace('>', '>') .replace('<', '<') @@ -40,6 +41,6 @@ def soft_unicode(s): """Make a string unicode if it isn't already. That way a markup string is not converted back to unicode. """ - if not isinstance(s, unicode): - s = unicode(s) + if not isinstance(s, six.text_type): + s = six.text_type(s) return s diff --git a/jinja2/_markupsafe/tests.py b/jinja2/_markupsafe/tests.py index c1ce394..449df49 100644 --- a/jinja2/_markupsafe/tests.py +++ b/jinja2/_markupsafe/tests.py @@ -1,6 +1,7 @@ import gc import unittest from jinja2._markupsafe import Markup, escape, escape_silent +import six class MarkupTestCase(unittest.TestCase): @@ -9,7 +10,7 @@ class MarkupTestCase(unittest.TestCase): # adding two strings should escape the unsafe one unsafe = '<script type="application/x-some-script">alert("foo");</script>' safe = Markup('<em>username</em>') - assert unsafe + safe == unicode(escape(unsafe)) + unicode(safe) + assert unsafe + safe == six.text_type(escape(unsafe)) + six.text_type(safe) # string interpolations are safe to use too assert Markup('<em>%s</em>') % '<bad user>' == \ @@ -55,8 +56,8 @@ class MarkupLeakTestCase(unittest.TestCase): def test_markup_leaks(self): counts = set() - for count in xrange(20): - for item in xrange(1000): + for count in range(20): + for item in range(1000): escape("foo") escape("<foo>") escape(u"foo") diff --git a/jinja2/_stringdefs.py b/jinja2/_stringdefs.py index 1161b7f..da5830e 100644 --- a/jinja2/_stringdefs.py +++ b/jinja2/_stringdefs.py @@ -13,6 +13,8 @@ :license: BSD, see LICENSE for details. """ +from jinja2._compat import unichr + Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' Cf = u'\xad\u0600\u0601\u0602\u0603\u06dd\u070f\u17b4\u17b5\u200b\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2060\u2061\u2062\u2063\u206a\u206b\u206c\u206d\u206e\u206f\ufeff\ufff9\ufffa\ufffb' diff --git a/jinja2/bccache.py b/jinja2/bccache.py index 0b0ccad..f022918 100644 --- a/jinja2/bccache.py +++ b/jinja2/bccache.py @@ -18,7 +18,8 @@ from os import path, listdir import sys import marshal import tempfile -import cPickle as pickle +from six.moves import cPickle as pickle +from six import BytesIO import fnmatch try: from hashlib import sha1 @@ -28,12 +29,10 @@ from jinja2.utils import open_if_exists # marshal works better on 3.x, one hack less required -if sys.version_info > (3, 0): - from io import BytesIO +if sys.version_info[0] >= 3: marshal_dump = marshal.dump marshal_load = marshal.load else: - from cStringIO import StringIO as BytesIO def marshal_dump(code, f): if isinstance(f, file): @@ -282,15 +281,26 @@ class MemcachedBytecodeCache(BytecodeCache): This bytecode cache does not support clearing of used items in the cache. The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. """ - def __init__(self, client, prefix='jinja2/bytecode/', timeout=None): + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, + ignore_memcache_errors=True): self.client = client self.prefix = prefix self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors def load_bytecode(self, bucket): - code = self.client.get(self.prefix + bucket.key) + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None if code is not None: bucket.bytecode_from_string(code) @@ -298,4 +308,8 @@ class MemcachedBytecodeCache(BytecodeCache): args = (self.prefix + bucket.key, bucket.bytecode_to_string()) if self.timeout is not None: args += (self.timeout,) - self.client.set(*args) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/jinja2/compiler.py b/jinja2/compiler.py index b21cb38..02df8c5 100644 --- a/jinja2/compiler.py +++ b/jinja2/compiler.py @@ -8,14 +8,16 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ -from cStringIO import StringIO from itertools import chain from copy import deepcopy from jinja2 import nodes from jinja2.nodes import EvalContext from jinja2.visitor import NodeVisitor from jinja2.exceptions import TemplateAssertionError -from jinja2.utils import Markup, concat, escape, is_python_keyword, next +from jinja2.utils import Markup, concat, escape, is_python_keyword +from jinja2._compat import range_type +import six +from six.moves import cStringIO as StringIO, map operators = { @@ -29,14 +31,6 @@ operators = { 'notin': 'not in' } -try: - exec '(0 if 0 else 0)' -except SyntaxError: - have_condexpr = False -else: - have_condexpr = True - - # what method to iterate over items do we want to use for dict iteration # in generated code? on 2.x let's go with iteritems, on 3.x with items if hasattr(dict, 'iteritems'): @@ -51,7 +45,11 @@ def unoptimize_before_dead_code(): def f(): if 0: dummy(x) return f -unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure) + +# The getattr is necessary for pypy which does not set this attribute if +# no closure is on the function +unoptimize_before_dead_code = bool( + getattr(unoptimize_before_dead_code(), '__closure__', None)) def generate(node, environment, name, filename, stream=None, @@ -69,8 +67,8 @@ def has_safe_repr(value): """Does the node have a safe representation?""" if value is None or value is NotImplemented or value is Ellipsis: return True - if isinstance(value, (bool, int, long, float, complex, basestring, - xrange, Markup)): + if isinstance(value, (bool, int, float, complex, range_type, + Markup) + six.string_types): return True if isinstance(value, (tuple, list, set, frozenset)): for item in value: @@ -78,7 +76,7 @@ def has_safe_repr(value): return False return True elif isinstance(value, dict): - for key, value in value.iteritems(): + for key, value in six.iteritems(value): if not has_safe_repr(key): return False if not has_safe_repr(value): @@ -542,7 +540,7 @@ class CodeGenerator(NodeVisitor): self.write(', ') self.visit(kwarg, frame) if extra_kwargs is not None: - for key, value in extra_kwargs.iteritems(): + for key, value in six.iteritems(extra_kwargs): self.write(', %s=%s' % (key, value)) if node.dyn_args: self.write(', *') @@ -558,7 +556,7 @@ class CodeGenerator(NodeVisitor): self.visit(kwarg.value, frame) self.write(', ') if extra_kwargs is not None: - for key, value in extra_kwargs.iteritems(): + for key, value in six.iteritems(extra_kwargs): self.write('%r: %s, ' % (key, value)) if node.dyn_kwargs is not None: self.write('}, **') @@ -625,7 +623,7 @@ class CodeGenerator(NodeVisitor): def pop_scope(self, aliases, frame): """Restore all aliases and delete unused variables.""" - for name, alias in aliases.iteritems(): + for name, alias in six.iteritems(aliases): self.writeline('l_%s = %s' % (name, alias)) to_delete = set() for name in frame.identifiers.declared_locally: @@ -663,16 +661,16 @@ class CodeGenerator(NodeVisitor): # it without aliasing all the variables. # this could be fixed in Python 3 where we have the nonlocal # keyword or if we switch to bytecode generation - overriden_closure_vars = ( + overridden_closure_vars = ( func_frame.identifiers.undeclared & func_frame.identifiers.declared & (func_frame.identifiers.declared_locally | func_frame.identifiers.declared_parameter) ) - if overriden_closure_vars: + if overridden_closure_vars: self.fail('It\'s not possible to set and access variables ' 'derived from an outer scope! (affects: %s)' % - ', '.join(sorted(overriden_closure_vars)), node.lineno) + ', '.join(sorted(overridden_closure_vars)), node.lineno) # remove variables from a closure from the frame's undeclared # identifiers. @@ -827,7 +825,7 @@ class CodeGenerator(NodeVisitor): self.outdent(2 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. - for name, block in self.blocks.iteritems(): + for name, block in six.iteritems(self.blocks): block_frame = Frame(eval_ctx) block_frame.inspect(block.body) block_frame.block = name @@ -930,7 +928,7 @@ class CodeGenerator(NodeVisitor): func_name = 'get_or_select_template' if isinstance(node.template, nodes.Const): - if isinstance(node.template.value, basestring): + if isinstance(node.template.value, six.string_types): func_name = 'get_template' elif isinstance(node.template.value, (tuple, list)): func_name = 'select_template' @@ -1068,6 +1066,7 @@ class CodeGenerator(NodeVisitor): # make sure the loop variable is a special one and raise a template # assertion error if a loop tries to write to loop if extended_loop: + self.writeline('l_loop = missing') loop_frame.identifiers.add_special('loop') for name in node.find_all(nodes.Name): if name.ctx == 'store' and name.name == 'loop': @@ -1216,9 +1215,9 @@ class CodeGenerator(NodeVisitor): return if self.environment.finalize: - finalize = lambda x: unicode(self.environment.finalize(x)) + finalize = lambda x: six.text_type(self.environment.finalize(x)) else: - finalize = unicode + finalize = six.text_type # if we are inside a frame that requires output checking, we do so outdent_later = False @@ -1352,7 +1351,7 @@ class CodeGenerator(NodeVisitor): public_names = [x for x in assignment_frame.toplevel_assignments if not x.startswith('_')] if len(assignment_frame.toplevel_assignments) == 1: - name = next(iter(assignment_frame.toplevel_assignments)) + name = six.advance_iterator(iter(assignment_frame.toplevel_assignments)) self.writeline('context.vars[%r] = l_%s' % (name, name)) else: self.writeline('context.vars.update({') @@ -1555,22 +1554,13 @@ class CodeGenerator(NodeVisitor): 'expression on %s evaluated to false and ' 'no else section was defined.' % self.position(node))) - if not have_condexpr: - self.write('((') - self.visit(node.test, frame) - self.write(') and (') - self.visit(node.expr1, frame) - self.write(',) or (') - write_expr2() - self.write(',))[0]') - else: - self.write('(') - self.visit(node.expr1, frame) - self.write(' if ') - self.visit(node.test, frame) - self.write(' else ') - write_expr2() - self.write(')') + self.write('(') + self.visit(node.expr1, frame) + self.write(' if ') + self.visit(node.test, frame) + self.write(' else ') + write_expr2() + self.write(')') def visit_Call(self, node, frame, forward_caller=False): if self.environment.sandboxed: diff --git a/jinja2/debug.py b/jinja2/debug.py index 3ac4041..31c25cc 100644 --- a/jinja2/debug.py +++ b/jinja2/debug.py @@ -15,6 +15,7 @@ import traceback from types import TracebackType from jinja2.utils import CodeType, missing, internal_code from jinja2.exceptions import TemplateSyntaxError +import six # on pypy we can take advantage of transparent proxies try: @@ -25,7 +26,7 @@ except ImportError: # how does the raise helper look like? try: - exec "raise TypeError, 'foo'" + exec("raise TypeError, 'foo'") except SyntaxError: raise_helper = 'raise __jinja_exception__[1]' except TypeError: @@ -158,7 +159,7 @@ def translate_exception(exc_info, initial_skip=0): frames = [] # skip some internal frames if wanted - for x in xrange(initial_skip): + for x in range(initial_skip): if tb is not None: tb = tb.tb_next initial_tb = tb @@ -189,7 +190,7 @@ def translate_exception(exc_info, initial_skip=0): # reraise it unchanged. # XXX: can we backup here? when could this happen? if not frames: - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) return ProcessedTraceback(exc_info[0], exc_info[1], frames) @@ -206,7 +207,7 @@ def fake_exc_info(exc_info, filename, lineno): locals = ctx.get_all() else: locals = {} - for name, value in real_locals.iteritems(): + for name, value in six.iteritems(real_locals): if name.startswith('l_') and value is not missing: locals[name[2:]] = value @@ -254,7 +255,7 @@ def fake_exc_info(exc_info, filename, lineno): # execute the code and catch the new traceback try: - exec code in globals, locals + exec(code, globals, locals) except: exc_info = sys.exc_info() new_tb = exc_info[2].tb_next @@ -330,10 +331,7 @@ def _init_ugly_crap(): tb_set_next = None if tproxy is None: try: - from jinja2._debugsupport import tb_set_next - except ImportError: - try: - tb_set_next = _init_ugly_crap() - except: - pass + tb_set_next = _init_ugly_crap() + except: + pass del _init_ugly_crap diff --git a/jinja2/defaults.py b/jinja2/defaults.py index d2d4544..a27cb80 100644 --- a/jinja2/defaults.py +++ b/jinja2/defaults.py @@ -8,6 +8,7 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ +from jinja2._compat import range_type from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner @@ -21,14 +22,16 @@ COMMENT_END_STRING = '#}' LINE_STATEMENT_PREFIX = None LINE_COMMENT_PREFIX = None TRIM_BLOCKS = False +LSTRIP_BLOCKS = False NEWLINE_SEQUENCE = '\n' +KEEP_TRAILING_NEWLINE = False # default filters, tests and namespace from jinja2.filters import FILTERS as DEFAULT_FILTERS from jinja2.tests import TESTS as DEFAULT_TESTS DEFAULT_NAMESPACE = { - 'range': xrange, + 'range': range_type, 'dict': lambda **kw: kw, 'lipsum': generate_lorem_ipsum, 'cycler': Cycler, diff --git a/jinja2/environment.py b/jinja2/environment.py index ebb5454..450cac1 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -11,16 +11,25 @@ import os import sys from jinja2 import nodes -from jinja2.defaults import * +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ + KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS from jinja2.lexer import get_lexer, TokenStream from jinja2.parser import Parser +from jinja2.nodes import EvalContext from jinja2.optimizer import optimize from jinja2.compiler import generate from jinja2.runtime import Undefined, new_context from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ - TemplatesNotFound + TemplatesNotFound, TemplateRuntimeError from jinja2.utils import import_string, LRUCache, Markup, missing, \ concat, consume, internalcode, _encode_filename +import six +from functools import reduce +from six.moves import filter, map # for direct template usage we have up to ten living environments @@ -71,7 +80,7 @@ def load_extensions(environment, extensions): """ result = {} for extension in extensions: - if isinstance(extension, basestring): + if isinstance(extension, six.string_types): extension = import_string(extension) result[extension.identifier] = extension(environment) return result @@ -134,12 +143,23 @@ class Environment(object): If this is set to ``True`` the first newline after a block is removed (block, not variable tag!). Defaults to `False`. + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + `newline_sequence` The sequence that starts a newline. Must be one of ``'\r'``, ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a useful default for Linux and OS X systems as well as web applications. + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + `extensions` List of Jinja extensions to use. This can either be import paths as strings or extension classes. For more information have a @@ -224,7 +244,9 @@ class Environment(object): line_statement_prefix=LINE_STATEMENT_PREFIX, line_comment_prefix=LINE_COMMENT_PREFIX, trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, extensions=(), optimized=True, undefined=Undefined, @@ -255,7 +277,9 @@ class Environment(object): self.line_statement_prefix = line_statement_prefix self.line_comment_prefix = line_comment_prefix self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline # runtime information self.undefined = undefined @@ -270,7 +294,6 @@ class Environment(object): # set the loader provided self.loader = loader - self.bytecode_cache = None self.cache = create_cache(cache_size) self.bytecode_cache = bytecode_cache self.auto_reload = auto_reload @@ -292,7 +315,7 @@ class Environment(object): yet. This is used by :ref:`extensions <writing-extensions>` to register callbacks and configuration values without breaking inheritance. """ - for key, value in attributes.iteritems(): + for key, value in six.iteritems(attributes): if not hasattr(self, key): setattr(self, key, value) @@ -300,7 +323,8 @@ class Environment(object): variable_start_string=missing, variable_end_string=missing, comment_start_string=missing, comment_end_string=missing, line_statement_prefix=missing, line_comment_prefix=missing, - trim_blocks=missing, extensions=missing, optimized=missing, + trim_blocks=missing, lstrip_blocks=missing, + extensions=missing, optimized=missing, undefined=missing, finalize=missing, autoescape=missing, loader=missing, cache_size=missing, auto_reload=missing, bytecode_cache=missing): @@ -323,7 +347,7 @@ class Environment(object): rv.overlayed = True rv.linked_to = self - for key, value in args.iteritems(): + for key, value in six.iteritems(args): if value is not missing: setattr(rv, key, value) @@ -333,7 +357,7 @@ class Environment(object): rv.cache = copy_cache(self.cache) rv.extensions = {} - for key, value in self.extensions.iteritems(): + for key, value in six.iteritems(self.extensions): rv.extensions[key] = value.bind(rv) if extensions is not missing: rv.extensions.update(load_extensions(rv, extensions)) @@ -352,7 +376,7 @@ class Environment(object): try: return obj[argument] except (TypeError, LookupError): - if isinstance(argument, basestring): + if isinstance(argument, six.string_types): try: attr = str(argument) except Exception: @@ -377,6 +401,42 @@ class Environment(object): except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) + def call_filter(self, name, value, args=None, kwargs=None, + context=None, eval_ctx=None): + """Invokes a filter on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + raise TemplateRuntimeError('no filter named %r' % name) + args = list(args or ()) + if getattr(func, 'contextfilter', False): + if context is None: + raise TemplateRuntimeError('Attempted to invoke context ' + 'filter without context') + args.insert(0, context) + elif getattr(func, 'evalcontextfilter', False): + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, 'environmentfilter', False): + args.insert(0, self) + return func(value, *args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + raise TemplateRuntimeError('no test named %r' % name) + return func(value, *(args or ()), **(kwargs or {})) + @internalcode def parse(self, source, name=None, filename=None): """Parse the sourcecode and return the abstract syntax tree. This @@ -407,7 +467,7 @@ class Environment(object): of the extensions to be applied you have to filter source through the :meth:`preprocess` method. """ - source = unicode(source) + source = six.text_type(source) try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: @@ -420,7 +480,7 @@ class Environment(object): because there you usually only want the actual source tokenized. """ return reduce(lambda s, e: e.preprocess(s, name, filename), - self.iter_extensions(), unicode(source)) + self.iter_extensions(), six.text_type(source)) def _tokenize(self, source, name, filename=None, state=None): """Called by the parser to do the preprocessing and filtering @@ -474,7 +534,7 @@ class Environment(object): """ source_hint = None try: - if isinstance(source, basestring): + if isinstance(source, six.string_types): source_hint = source source = self._parse(source, name, filename) if self.optimized: @@ -570,10 +630,14 @@ class Environment(object): py_header = imp.get_magic() + \ u'\xff\xff\xff\xff'.encode('iso-8859-15') + # Python 3.3 added a source filesize to the header + if sys.version_info >= (3, 3): + py_header += u'\x00\x00\x00\x00'.encode('iso-8859-15') + def write_file(filename, data, mode): if zip: info = ZipInfo(filename) - info.external_attr = 0755 << 16L + info.external_attr = 0o755 << 16 zip_file.writestr(info, data) else: f = open(os.path.join(target, filename), mode) @@ -597,7 +661,7 @@ class Environment(object): source, filename, _ = self.loader.get_source(self, name) try: code = self.compile(source, name, filename, True, True) - except TemplateSyntaxError, e: + except TemplateSyntaxError as e: if not ignore_errors: raise log_function('Could not compile "%s": %s' % (name, e)) @@ -667,7 +731,7 @@ class Environment(object): if self.exception_handler is not None: self.exception_handler(traceback) exc_type, exc_value, tb = traceback.standard_exc_info - raise exc_type, exc_value, tb + six.reraise(exc_type, exc_value, tb) def join_path(self, template, parent): """Join a template with the parent. By default all the lookups are @@ -754,7 +818,7 @@ class Environment(object): .. versionadded:: 2.3 """ - if isinstance(template_name_or_list, basestring): + if isinstance(template_name_or_list, six.string_types): return self.get_template(template_name_or_list, parent, globals) elif isinstance(template_name_or_list, Template): return template_name_or_list @@ -816,7 +880,9 @@ class Template(object): line_statement_prefix=LINE_STATEMENT_PREFIX, line_comment_prefix=LINE_COMMENT_PREFIX, trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, extensions=(), optimized=True, undefined=Undefined, @@ -826,8 +892,9 @@ class Template(object): block_start_string, block_end_string, variable_start_string, variable_end_string, comment_start_string, comment_end_string, line_statement_prefix, line_comment_prefix, trim_blocks, - newline_sequence, frozenset(extensions), optimized, undefined, - finalize, autoescape, None, 0, False, None) + lstrip_blocks, newline_sequence, keep_trailing_newline, + frozenset(extensions), optimized, undefined, finalize, autoescape, + None, 0, False, None) return env.from_string(source, template_class=cls) @classmethod @@ -839,7 +906,7 @@ class Template(object): 'environment': environment, '__file__': code.co_filename } - exec code in namespace + exec(code, namespace) rv = cls._from_namespace(environment, namespace, globals) rv._uptodate = uptodate return rv @@ -999,12 +1066,9 @@ class TemplateModule(object): return Markup(concat(self._body_stream)) def __str__(self): - return unicode(self).encode('utf-8') + s = self.__unicode__() + return s if six.PY3 else s.encode('utf-8') - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. def __unicode__(self): return concat(self._body_stream) @@ -1035,7 +1099,7 @@ class TemplateExpression(object): return rv -class TemplateStream(object): +class TemplateStream(six.Iterator): """A template stream works pretty much like an ordinary python generator but it can buffer multiple items to reduce the number of total iterations. Per default the output is unbuffered which means that for every unbuffered @@ -1060,8 +1124,8 @@ class TemplateStream(object): Template('Hello {{ name }}!').stream(name='foo').dump('hello.html') """ close = False - if isinstance(fp, basestring): - fp = file(fp, 'w') + if isinstance(fp, six.string_types): + fp = open(fp, encoding is None and 'w' or 'wb') close = True try: if encoding is not None: @@ -1079,7 +1143,7 @@ class TemplateStream(object): def disable_buffering(self): """Disable the output buffering.""" - self._next = self._gen.next + self._next = lambda: six.next(self._gen) self.buffered = False def enable_buffering(self, size=5): @@ -1107,12 +1171,12 @@ class TemplateStream(object): c_size = 0 self.buffered = True - self._next = generator(self._gen.next).next + self._next = lambda: six.next(generator(lambda: six.next(self._gen))) def __iter__(self): return self - def next(self): + def __next__(self): return self._next() diff --git a/jinja2/exceptions.py b/jinja2/exceptions.py index 771f6a8..9fe698b 100644 --- a/jinja2/exceptions.py +++ b/jinja2/exceptions.py @@ -8,22 +8,34 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ +import sys +import six +from six.moves import map class TemplateError(Exception): """Baseclass for all template errors.""" - def __init__(self, message=None): - if message is not None: - message = unicode(message).encode('utf-8') - Exception.__init__(self, message) - - @property - def message(self): - if self.args: - message = self.args[0] + if sys.version_info[0] < 3: + def __init__(self, message=None): if message is not None: - return message.decode('utf-8', 'replace') + message = six.text_type(message).encode('utf-8') + Exception.__init__(self, message) + + @property + def message(self): + if self.args: + message = self.args[0] + if message is not None: + return message.decode('utf-8', 'replace') + + else: + @property + def message(self): + if self.args: + message = self.args[0] + if message is not None: + return message class TemplateNotFound(IOError, LookupError, TemplateError): @@ -62,8 +74,8 @@ class TemplatesNotFound(TemplateNotFound): def __init__(self, names=(), message=None): if message is None: - message = u'non of the templates given were found: ' + \ - u', '.join(map(unicode, names)) + message = u'none of the templates given were found: ' + \ + u', '.join(map(six.text_type, names)) TemplateNotFound.__init__(self, names and names[-1] or None, message) self.templates = list(names) @@ -83,12 +95,9 @@ class TemplateSyntaxError(TemplateError): self.translated = False def __str__(self): - return unicode(self).encode('utf-8') + s = self.__unicode__() + return s if six.PY3 else s.encode('utf-8') - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. def __unicode__(self): # for translated errors we only return the message if self.translated: diff --git a/jinja2/ext.py b/jinja2/ext.py index 78d2429..c456dbf 100644 --- a/jinja2/ext.py +++ b/jinja2/ext.py @@ -10,13 +10,17 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD. """ -from collections import deque from jinja2 import nodes -from jinja2.defaults import * +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS from jinja2.environment import Environment -from jinja2.runtime import Undefined, concat +from jinja2.runtime import concat from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError -from jinja2.utils import contextfunction, import_string, Markup, next +from jinja2.utils import contextfunction, import_string, Markup +import six # the only real useful gettext functions for a Jinja template. Note @@ -34,7 +38,7 @@ class ExtensionRegistry(type): return rv -class Extension(object): +class Extension(six.with_metaclass(ExtensionRegistry, object)): """Extensions can be used to add extra functionality to the Jinja template system at the parser level. Custom extensions are bound to an environment but may not store environment specific data on `self`. The reason for @@ -205,13 +209,13 @@ class InternationalizationExtension(Extension): self.environment.globals.pop(key, None) def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): - if isinstance(source, basestring): + if isinstance(source, six.string_types): source = self.environment.parse(source) return extract_from_ast(source, gettext_functions) def parse(self, parser): """Parse a translatable tag.""" - lineno = next(parser.stream).lineno + lineno = six.advance_iterator(parser.stream).lineno num_called_num = False # find all the variables referenced. Additionally a variable can be @@ -236,7 +240,7 @@ class InternationalizationExtension(Extension): # expressions if parser.stream.current.type == 'assign': - next(parser.stream) + six.advance_iterator(parser.stream) variables[name.value] = var = parser.parse_expression() else: variables[name.value] = var = nodes.Name(name.value, 'load') @@ -268,7 +272,7 @@ class InternationalizationExtension(Extension): # if we have a pluralize block, we parse that too if parser.stream.current.test('name:pluralize'): have_plural = True - next(parser.stream) + six.advance_iterator(parser.stream) if parser.stream.current.type != 'block_end': name = parser.stream.expect('name') if name.value not in variables: @@ -279,10 +283,10 @@ class InternationalizationExtension(Extension): num_called_num = name.value == 'num' parser.stream.expect('block_end') plural_names, plural = self._parse_block(parser, False) - next(parser.stream) + six.advance_iterator(parser.stream) referenced.update(plural_names) else: - next(parser.stream) + six.advance_iterator(parser.stream) # register free names as simple name expressions for var in referenced: @@ -310,15 +314,15 @@ class InternationalizationExtension(Extension): while 1: if parser.stream.current.type == 'data': buf.append(parser.stream.current.value.replace('%', '%%')) - next(parser.stream) + six.advance_iterator(parser.stream) elif parser.stream.current.type == 'variable_begin': - next(parser.stream) + six.advance_iterator(parser.stream) name = parser.stream.expect('name').value referenced.append(name) buf.append('%%(%s)s' % name) parser.stream.expect('variable_end') elif parser.stream.current.type == 'block_begin': - next(parser.stream) + six.advance_iterator(parser.stream) if parser.stream.current.test('name:endtrans'): break elif parser.stream.current.test('name:pluralize'): @@ -364,7 +368,7 @@ class InternationalizationExtension(Extension): # enough to handle the variable expansion and autoescape # handling itself if self.environment.newstyle_gettext: - for key, value in variables.iteritems(): + for key, value in six.iteritems(variables): # the function adds that later anyways in case num was # called num, so just skip it. if num_called_num and key == 'num': @@ -391,7 +395,7 @@ class ExprStmtExtension(Extension): tags = set(['do']) def parse(self, parser): - node = nodes.ExprStmt(lineno=next(parser.stream).lineno) + node = nodes.ExprStmt(lineno=six.advance_iterator(parser.stream).lineno) node.node = parser.parse_tuple() return node @@ -401,7 +405,7 @@ class LoopControlExtension(Extension): tags = set(['break', 'continue']) def parse(self, parser): - token = next(parser.stream) + token = six.advance_iterator(parser.stream) if token.value == 'break': return nodes.Break(lineno=token.lineno) return nodes.Continue(lineno=token.lineno) @@ -412,7 +416,7 @@ class WithExtension(Extension): tags = set(['with']) def parse(self, parser): - node = nodes.Scope(lineno=next(parser.stream).lineno) + node = nodes.Scope(lineno=six.advance_iterator(parser.stream).lineno) assignments = [] while parser.stream.current.type != 'block_end': lineno = parser.stream.current.lineno @@ -433,7 +437,7 @@ class AutoEscapeExtension(Extension): tags = set(['autoescape']) def parse(self, parser): - node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno) + node = nodes.ScopedEvalContextModifier(lineno=six.advance_iterator(parser.stream).lineno) node.options = [ nodes.Keyword('autoescape', parser.parse_expression()) ] @@ -486,7 +490,7 @@ def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, strings = [] for arg in node.args: if isinstance(arg, nodes.Const) and \ - isinstance(arg.value, basestring): + isinstance(arg.value, six.string_types): strings.append(arg.value) else: strings.append(None) @@ -599,7 +603,10 @@ def babel_extract(fileobj, keywords, comment_tags, options): options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, getbool(options, 'trim_blocks', TRIM_BLOCKS), - NEWLINE_SEQUENCE, frozenset(extensions), + getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS), + NEWLINE_SEQUENCE, + getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), + frozenset(extensions), cache_size=0, auto_reload=False ) @@ -611,7 +618,7 @@ def babel_extract(fileobj, keywords, comment_tags, options): try: node = environment.parse(source) tokens = list(environment.lex(environment.preprocess(source))) - except TemplateSyntaxError, e: + except TemplateSyntaxError as e: if not silent: raise # skip templates with syntax errors diff --git a/jinja2/filters.py b/jinja2/filters.py index 8dd6ff0..adc513e 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -12,11 +12,13 @@ import re import math from random import choice from operator import itemgetter -from itertools import imap, groupby +from itertools import groupby from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ unicode_urlencode from jinja2.runtime import Undefined from jinja2.exceptions import FilterArgumentError +import six +from six.moves import map _word_re = re.compile(r'\w+(?u)') @@ -54,7 +56,7 @@ def make_attrgetter(environment, attribute): passed object with the rules of the environment. Dots are allowed to access attributes of attributes. """ - if not isinstance(attribute, basestring) or '.' not in attribute: + if not isinstance(attribute, six.string_types) or '.' not in attribute: return lambda x: environment.getitem(x, attribute) attribute = attribute.split('.') def attrgetter(item): @@ -68,7 +70,7 @@ def do_forceescape(value): """Enforce HTML escaping. This will probably double escape variables.""" if hasattr(value, '__html__'): value = value.__html__() - return escape(unicode(value)) + return escape(six.text_type(value)) def do_urlencode(value): @@ -79,8 +81,8 @@ def do_urlencode(value): """ itemiter = None if isinstance(value, dict): - itemiter = value.iteritems() - elif not isinstance(value, basestring): + itemiter = six.iteritems(value) + elif not isinstance(value, six.string_types): try: itemiter = iter(value) except TypeError: @@ -110,7 +112,7 @@ def do_replace(eval_ctx, s, old, new, count=None): if count is None: count = -1 if not eval_ctx.autoescape: - return unicode(s).replace(unicode(old), unicode(new), count) + return six.text_type(s).replace(six.text_type(old), six.text_type(new), count) if hasattr(old, '__html__') or hasattr(new, '__html__') and \ not hasattr(s, '__html__'): s = escape(s) @@ -155,7 +157,7 @@ def do_xmlattr(_eval_ctx, d, autospace=True): """ rv = u' '.join( u'%s="%s"' % (escape(key), escape(value)) - for key, value in d.iteritems() + for key, value in six.iteritems(d) if value is not None and not isinstance(value, Undefined) ) if autospace and rv: @@ -176,7 +178,12 @@ def do_title(s): """Return a titlecased version of the value. I.e. words will start with uppercase letters, all remaining characters are lowercase. """ - return soft_unicode(s).title() + rv = [] + for item in re.compile(r'([-\s]+)(?u)').split(s): + if not item: + continue + rv.append(item[0].upper() + item[1:]) + return ''.join(rv) def do_dictsort(value, case_sensitive=False, by='key'): @@ -189,7 +196,7 @@ def do_dictsort(value, case_sensitive=False, by='key'): {% for item in mydict|dictsort %} sort the dict by key, case insensitive - {% for item in mydict|dicsort(true) %} + {% for item in mydict|dictsort(true) %} sort the dict by key, case sensitive {% for item in mydict|dictsort(false, 'value') %} @@ -205,7 +212,7 @@ def do_dictsort(value, case_sensitive=False, by='key'): '"key" or "value"') def sort_func(item): value = item[pos] - if isinstance(value, basestring) and not case_sensitive: + if isinstance(value, six.string_types) and not case_sensitive: value = value.lower() return value @@ -242,7 +249,7 @@ def do_sort(environment, value, reverse=False, case_sensitive=False, """ if not case_sensitive: def sort_func(item): - if isinstance(item, basestring): + if isinstance(item, six.string_types): item = item.lower() return item else: @@ -271,7 +278,7 @@ def do_default(value, default_value=u'', boolean=False): {{ ''|default('the string was empty', true) }} """ - if (boolean and not value) or isinstance(value, Undefined): + if isinstance(value, Undefined) or (boolean and not value): return default_value return value @@ -300,11 +307,11 @@ def do_join(eval_ctx, value, d=u'', attribute=None): The `attribute` parameter was added. """ if attribute is not None: - value = imap(make_attrgetter(eval_ctx.environment, attribute), value) + value = map(make_attrgetter(eval_ctx.environment, attribute), value) # no automatic escaping? joining is a lot eaiser then if not eval_ctx.autoescape: - return unicode(d).join(imap(unicode, value)) + return six.text_type(d).join(map(six.text_type, value)) # if the delimiter doesn't have an html representation we check # if any of the items has. If yes we do a coercion to Markup @@ -315,27 +322,27 @@ def do_join(eval_ctx, value, d=u'', attribute=None): if hasattr(item, '__html__'): do_escape = True else: - value[idx] = unicode(item) + value[idx] = six.text_type(item) if do_escape: d = escape(d) else: - d = unicode(d) + d = six.text_type(d) return d.join(value) # no html involved, to normal joining - return soft_unicode(d).join(imap(soft_unicode, value)) + return soft_unicode(d).join(map(soft_unicode, value)) def do_center(value, width=80): """Centers the value in a field of a given width.""" - return unicode(value).center(width) + return six.text_type(value).center(width) @environmentfilter def do_first(environment, seq): """Return the first item of a sequence.""" try: - return iter(seq).next() + return six.advance_iterator(iter(seq)) except StopIteration: return environment.undefined('No first item, sequence was empty.') @@ -344,7 +351,7 @@ def do_first(environment, seq): def do_last(environment, seq): """Return the last item of a sequence.""" try: - return iter(reversed(seq)).next() + return six.advance_iterator(iter(reversed(seq))) except StopIteration: return environment.undefined('No last item, sequence was empty.') @@ -438,16 +445,17 @@ def do_truncate(s, length=255, killwords=False, end='...'): """Return a truncated copy of the string. The length is specified with the first parameter which defaults to ``255``. If the second parameter is ``true`` the filter will cut the text at length. Otherwise - it will try to save the last word. If the text was in fact + it will discard the last word. If the text was in fact truncated it will append an ellipsis sign (``"..."``). If you want a different ellipsis sign than ``"..."`` you can specify it using the third parameter. - .. sourcecode jinja:: + .. sourcecode:: jinja - {{ mytext|truncate(300, false, '»') }} - truncate mytext to 300 chars, don't split up words, use a - right pointing double arrow as ellipsis sign. + {{ "foo bar"|truncate(5) }} + -> "foo ..." + {{ "foo bar"|truncate(5, True) }} + -> "foo b..." """ if len(s) <= length: return s @@ -465,15 +473,23 @@ def do_truncate(s, length=255, killwords=False, end='...'): return u' '.join(result) @environmentfilter -def do_wordwrap(environment, s, width=79, break_long_words=True): +def do_wordwrap(environment, s, width=79, break_long_words=True, + wrapstring=None): """ Return a copy of the string passed to the filter wrapped after ``79`` characters. You can override this default using the first parameter. If you set the second parameter to `false` Jinja will not - split words apart if they are longer than `width`. + split words apart if they are longer than `width`. By default, the newlines + will be the default newlines for the environment, but this can be changed + using the wrapstring keyword argument. + + .. versionadded:: 2.7 + Added support for the `wrapstring` parameter. """ + if not wrapstring: + wrapstring = environment.newline_sequence import textwrap - return environment.newline_sequence.join(textwrap.wrap(s, width=width, expand_tabs=False, + return wrapstring.join(textwrap.wrap(s, width=width, expand_tabs=False, replace_whitespace=False, break_long_words=break_long_words)) @@ -534,7 +550,7 @@ def do_striptags(value): """ if hasattr(value, '__html__'): value = value.__html__() - return Markup(unicode(value)).striptags() + return Markup(six.text_type(value)).striptags() def do_slice(value, slices, fill_with=None): @@ -562,7 +578,7 @@ def do_slice(value, slices, fill_with=None): items_per_slice = length // slices slices_with_extra = length % slices offset = 0 - for slice_number in xrange(slices): + for slice_number in range(slices): start = offset + slice_number * items_per_slice if slice_number < slices_with_extra: offset += 1 @@ -578,7 +594,7 @@ def do_batch(value, linecount, fill_with=None): A filter that batches items. It works pretty much like `slice` just the other way round. It returns a list of lists with the given number of items. If you provide a second parameter this - is used to fill missing items. See this example: + is used to fill up missing items. See this example: .. sourcecode:: html+jinja @@ -687,7 +703,8 @@ class _GroupTuple(tuple): grouper = property(itemgetter(0)) list = property(itemgetter(1)) - def __new__(cls, (key, value)): + def __new__(cls, xxx_todo_changeme): + (key, value) = xxx_todo_changeme return tuple.__new__(cls, (key, list(value))) @@ -708,7 +725,7 @@ def do_sum(environment, iterable, attribute=None, start=0): attributes. Also the `start` parameter was moved on to the right. """ if attribute is not None: - iterable = imap(make_attrgetter(environment, attribute), iterable) + iterable = map(make_attrgetter(environment, attribute), iterable) return sum(iterable, start) @@ -728,14 +745,14 @@ def do_mark_safe(value): def do_mark_unsafe(value): """Mark a value as unsafe. This is the reverse operation for :func:`safe`.""" - return unicode(value) + return six.text_type(value) def do_reverse(value): """Reverse the object or return an iterator the iterates over it the other way round. """ - if isinstance(value, basestring): + if isinstance(value, six.string_types): return value[::-1] try: return reversed(value) @@ -773,6 +790,144 @@ def do_attr(environment, obj, name): return environment.undefined(obj=obj, name=name) +@contextfilter +def do_map(*args, **kwargs): + """Applies a filter on a sequence of objects or looks up an attribute. + This is useful when dealing with lists of objects but you are really + only interested in a certain value of it. + + The basic usage is mapping on an attribute. Imagine you have a list + of users but you are only interested in a list of usernames: + + .. sourcecode:: jinja + + Users on this page: {{ users|map(attribute='username')|join(', ') }} + + Alternatively you can let it invoke a filter by passing the name of the + filter and the arguments afterwards. A good example would be applying a + text conversion filter on a sequence: + + .. sourcecode:: jinja + + Users on this page: {{ titles|map('lower')|join(', ') }} + + .. versionadded:: 2.7 + """ + context = args[0] + seq = args[1] + + if len(args) == 2 and 'attribute' in kwargs: + attribute = kwargs.pop('attribute') + if kwargs: + raise FilterArgumentError('Unexpected keyword argument %r' % + six.advance_iterator(iter(kwargs))) + func = make_attrgetter(context.environment, attribute) + else: + try: + name = args[2] + args = args[3:] + except LookupError: + raise FilterArgumentError('map requires a filter argument') + func = lambda item: context.environment.call_filter( + name, item, args, kwargs, context=context) + + if seq: + for item in seq: + yield func(item) + + +@contextfilter +def do_select(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and only selecting the ones with the test succeeding. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|select("odd") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: x, False) + + +@contextfilter +def do_reject(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and rejecting the ones with the test succeeding. + + Example usage: + + .. sourcecode:: jinja + + {{ numbers|reject("odd") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: not x, False) + + +@contextfilter +def do_selectattr(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and only selecting the ones with the test succeeding. + + Example usage: + + .. sourcecode:: jinja + + {{ users|selectattr("is_active") }} + {{ users|selectattr("email", "none") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: x, True) + + +@contextfilter +def do_rejectattr(*args, **kwargs): + """Filters a sequence of objects by appying a test to either the object + or the attribute and rejecting the ones with the test succeeding. + + .. sourcecode:: jinja + + {{ users|rejectattr("is_active") }} + {{ users|rejectattr("email", "none") }} + + .. versionadded:: 2.7 + """ + return _select_or_reject(args, kwargs, lambda x: not x, True) + + +def _select_or_reject(args, kwargs, modfunc, lookup_attr): + context = args[0] + seq = args[1] + if lookup_attr: + try: + attr = args[2] + except LookupError: + raise FilterArgumentError('Missing parameter for attribute name') + transfunc = make_attrgetter(context.environment, attr) + off = 1 + else: + off = 0 + transfunc = lambda x: x + + try: + name = args[2 + off] + args = args[3 + off:] + func = lambda item: context.environment.call_test( + name, item, args, kwargs) + except LookupError: + func = bool + + if seq: + for item in seq: + if modfunc(func(transfunc(item))): + yield item + + FILTERS = { 'attr': do_attr, 'replace': do_replace, @@ -797,7 +952,10 @@ FILTERS = { 'capitalize': do_capitalize, 'first': do_first, 'last': do_last, + 'map': do_map, 'random': do_random, + 'reject': do_reject, + 'rejectattr': do_rejectattr, 'filesizeformat': do_filesizeformat, 'pprint': do_pprint, 'truncate': do_truncate, @@ -811,6 +969,8 @@ FILTERS = { 'format': do_format, 'trim': do_trim, 'striptags': do_striptags, + 'select': do_select, + 'selectattr': do_selectattr, 'slice': do_slice, 'batch': do_batch, 'sum': do_sum, diff --git a/jinja2/lexer.py b/jinja2/lexer.py index 69865d0..b39a738 100644 --- a/jinja2/lexer.py +++ b/jinja2/lexer.py @@ -18,7 +18,8 @@ import re from operator import itemgetter from collections import deque from jinja2.exceptions import TemplateSyntaxError -from jinja2.utils import LRUCache, next +from jinja2.utils import LRUCache +import six # cache for the lexers. Exists in order to be able to have multiple @@ -45,6 +46,12 @@ else: float_re = re.compile(r'(?<!\.)\d+\.\d+') newline_re = re.compile(r'(\r\n|\r|\n)') +try: + intern = intern # py2 +except NameError: + import sys + intern = sys.intern # py3 + # internal the tokens and keep references to them TOKEN_ADD = intern('add') TOKEN_ASSIGN = intern('assign') @@ -126,7 +133,7 @@ operators = { ';': TOKEN_SEMICOLON } -reverse_operators = dict([(v, k) for k, v in operators.iteritems()]) +reverse_operators = dict([(v, k) for k, v in six.iteritems(operators)]) assert len(operators) == len(reverse_operators), 'operators dropped' operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))) @@ -197,7 +204,7 @@ def compile_rules(environment): if environment.line_statement_prefix is not None: rules.append((len(environment.line_statement_prefix), 'linestatement', - r'^\s*' + e(environment.line_statement_prefix))) + r'^[ \t\v]*' + e(environment.line_statement_prefix))) if environment.line_comment_prefix is not None: rules.append((len(environment.line_comment_prefix), 'linecomment', r'(?:^|(?<=\S))[^\S\r\n]*' + @@ -262,7 +269,7 @@ class Token(tuple): ) -class TokenStreamIterator(object): +class TokenStreamIterator(six.Iterator): """The iterator for tokenstreams. Iterate over the stream until the eof token is reached. """ @@ -273,35 +280,36 @@ class TokenStreamIterator(object): def __iter__(self): return self - def next(self): + def __next__(self): token = self.stream.current if token.type is TOKEN_EOF: self.stream.close() raise StopIteration() - next(self.stream) + six.advance_iterator(self.stream) return token -class TokenStream(object): +class TokenStream(six.Iterator): """A token stream is an iterable that yields :class:`Token`\s. The parser however does not iterate over it but calls :meth:`next` to go one token ahead. The current active token is stored as :attr:`current`. """ def __init__(self, generator, name, filename): - self._next = iter(generator).next + self._iter = iter(generator) self._pushed = deque() self.name = name self.filename = filename self.closed = False self.current = Token(1, TOKEN_INITIAL, '') - next(self) + six.advance_iterator(self) def __iter__(self): return TokenStreamIterator(self) - def __nonzero__(self): + def __bool__(self): return bool(self._pushed) or self.current.type is not TOKEN_EOF + __nonzero__ = __bool__ # py2 eos = property(lambda x: not x, doc="Are we at the end of the stream?") @@ -311,7 +319,7 @@ class TokenStream(object): def look(self): """Look at the next token.""" - old_token = next(self) + old_token = six.advance_iterator(self) result = self.current self.push(result) self.current = old_token @@ -319,28 +327,28 @@ class TokenStream(object): def skip(self, n=1): """Got n tokens ahead.""" - for x in xrange(n): - next(self) + for x in range(n): + six.advance_iterator(self) def next_if(self, expr): """Perform the token test and return the token if it matched. Otherwise the return value is `None`. """ if self.current.test(expr): - return next(self) + return six.advance_iterator(self) def skip_if(self, expr): """Like :meth:`next_if` but only returns `True` or `False`.""" return self.next_if(expr) is not None - def next(self): + def __next__(self): """Go one token ahead and return the old one""" rv = self.current if self._pushed: self.current = self._pushed.popleft() elif self.current.type is not TOKEN_EOF: try: - self.current = self._next() + self.current = six.advance_iterator(self._iter) except StopIteration: self.close() return rv @@ -348,7 +356,7 @@ class TokenStream(object): def close(self): """Close the stream.""" self.current = Token(self.current.lineno, TOKEN_EOF, '') - self._next = None + self._iter = None self.closed = True def expect(self, expr): @@ -369,7 +377,7 @@ class TokenStream(object): try: return self.current finally: - next(self) + six.advance_iterator(self) def get_lexer(environment): @@ -383,7 +391,9 @@ def get_lexer(environment): environment.line_statement_prefix, environment.line_comment_prefix, environment.trim_blocks, - environment.newline_sequence) + environment.lstrip_blocks, + environment.newline_sequence, + environment.keep_trailing_newline) lexer = _lexer_cache.get(key) if lexer is None: lexer = Lexer(environment) @@ -425,7 +435,44 @@ class Lexer(object): # block suffix if trimming is enabled block_suffix_re = environment.trim_blocks and '\\n?' or '' + # strip leading spaces if lstrip_blocks is enabled + prefix_re = {} + if environment.lstrip_blocks: + # use '{%+' to manually disable lstrip_blocks behavior + no_lstrip_re = e('+') + # detect overlap between block and variable or comment strings + block_diff = c(r'^%s(.*)' % e(environment.block_start_string)) + # make sure we don't mistake a block for a variable or a comment + m = block_diff.match(environment.comment_start_string) + no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' + m = block_diff.match(environment.variable_start_string) + no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' + + # detect overlap between comment and variable strings + comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string)) + m = comment_diff.match(environment.variable_start_string) + no_variable_re = m and r'(?!%s)' % e(m.group(1)) or '' + + lstrip_re = r'^[ \t]*' + block_prefix_re = r'%s%s(?!%s)|%s\+?' % ( + lstrip_re, + e(environment.block_start_string), + no_lstrip_re, + e(environment.block_start_string), + ) + comment_prefix_re = r'%s%s%s|%s\+?' % ( + lstrip_re, + e(environment.comment_start_string), + no_variable_re, + e(environment.comment_start_string), + ) + prefix_re['block'] = block_prefix_re + prefix_re['comment'] = comment_prefix_re + else: + block_prefix_re = '%s' % e(environment.block_start_string) + self.newline_sequence = environment.newline_sequence + self.keep_trailing_newline = environment.keep_trailing_newline # global lexing rules self.rules = { @@ -434,11 +481,11 @@ class Lexer(object): (c('(.*?)(?:%s)' % '|'.join( [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( e(environment.block_start_string), - e(environment.block_start_string), + block_prefix_re, e(environment.block_end_string), e(environment.block_end_string) )] + [ - r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r) + r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r)) for n, r in root_tag_rules ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), # data @@ -472,7 +519,7 @@ class Lexer(object): TOKEN_RAW_BEGIN: [ (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( e(environment.block_start_string), - e(environment.block_start_string), + block_prefix_re, e(environment.block_end_string), e(environment.block_end_string), block_suffix_re @@ -526,7 +573,7 @@ class Lexer(object): value = self._normalize_newlines(value[1:-1]) \ .encode('ascii', 'backslashreplace') \ .decode('unicode-escape') - except Exception, e: + except Exception as e: msg = str(e).split(':')[-1].strip() raise TemplateSyntaxError(msg, lineno, name, filename) # if we can express it as bytestring (ascii only) @@ -549,7 +596,14 @@ class Lexer(object): """This method tokenizes the text and returns the tokens in a generator. Use this method if you just want to tokenize a template. """ - source = '\n'.join(unicode(source).splitlines()) + source = six.text_type(source) + lines = source.splitlines() + if self.keep_trailing_newline and source: + for newline in ('\r\n', '\r', '\n'): + if source.endswith(newline): + lines.append('') + break + source = '\n'.join(lines) pos = 0 lineno = 1 stack = ['root'] @@ -590,7 +644,7 @@ class Lexer(object): # yield for the current token the first named # group that matched elif token == '#bygroup': - for key, value in m.groupdict().iteritems(): + for key, value in six.iteritems(m.groupdict()): if value is not None: yield lineno, key, value lineno += value.count('\n') @@ -647,7 +701,7 @@ class Lexer(object): stack.pop() # resolve the new state by group checking elif new_state == '#bygroup': - for key, value in m.groupdict().iteritems(): + for key, value in six.iteritems(m.groupdict()): if value is not None: stack.append(key) break diff --git a/jinja2/loaders.py b/jinja2/loaders.py index c90bbe7..f8e4f99 100644 --- a/jinja2/loaders.py +++ b/jinja2/loaders.py @@ -13,12 +13,13 @@ import sys import weakref from types import ModuleType from os import path +import six try: from hashlib import sha1 except ImportError: from sha import new as sha1 from jinja2.exceptions import TemplateNotFound -from jinja2.utils import LRUCache, open_if_exists, internalcode +from jinja2.utils import open_if_exists, internalcode def split_template_path(template): @@ -153,7 +154,7 @@ class FileSystemLoader(BaseLoader): """ def __init__(self, searchpath, encoding='utf-8'): - if isinstance(searchpath, basestring): + if isinstance(searchpath, six.string_types): searchpath = [searchpath] self.searchpath = list(searchpath) self.encoding = encoding @@ -274,7 +275,7 @@ class DictLoader(BaseLoader): def get_source(self, environment, template): if template in self.mapping: source = self.mapping[template] - return source, None, lambda: source != self.mapping.get(template) + return source, None, lambda: source == self.mapping.get(template) raise TemplateNotFound(template) def list_templates(self): @@ -306,7 +307,7 @@ class FunctionLoader(BaseLoader): rv = self.load_func(template) if rv is None: raise TemplateNotFound(template) - elif isinstance(rv, basestring): + elif isinstance(rv, six.string_types): return rv, None, None return rv @@ -359,7 +360,7 @@ class PrefixLoader(BaseLoader): def list_templates(self): result = [] - for prefix, loader in self.mapping.iteritems(): + for prefix, loader in six.iteritems(self.mapping): for template in loader.list_templates(): result.append(prefix + self.delimiter + template) return result @@ -431,7 +432,7 @@ class ModuleLoader(BaseLoader): # create a fake module that looks for the templates in the # path given. mod = _TemplateModule(package_name) - if isinstance(path, basestring): + if isinstance(path, six.string_types): path = [path] else: path = list(path) diff --git a/jinja2/meta.py b/jinja2/meta.py index 3a779a5..26ae0b9 100644 --- a/jinja2/meta.py +++ b/jinja2/meta.py @@ -11,7 +11,7 @@ """ from jinja2 import nodes from jinja2.compiler import CodeGenerator - +import six class TrackingCodeGenerator(CodeGenerator): """We abuse the code generator for introspection.""" @@ -77,7 +77,7 @@ def find_referenced_templates(ast): # something const, only yield the strings and ignore # non-string consts that really just make no sense if isinstance(template_name, nodes.Const): - if isinstance(template_name.value, basestring): + if isinstance(template_name.value, six.string_types): yield template_name.value # something dynamic in there else: @@ -87,7 +87,7 @@ def find_referenced_templates(ast): yield None continue # constant is a basestring, direct template name - if isinstance(node.template.value, basestring): + if isinstance(node.template.value, six.string_types): yield node.template.value # a tuple or list (latter *should* not happen) made of consts, # yield the consts that are strings. We could warn here for @@ -95,7 +95,7 @@ def find_referenced_templates(ast): elif isinstance(node, nodes.Include) and \ isinstance(node.template.value, (tuple, list)): for template_name in node.template.value: - if isinstance(template_name, basestring): + if isinstance(template_name, six.string_types): yield template_name # something else we don't care about, we could warn here else: diff --git a/jinja2/nodes.py b/jinja2/nodes.py index f9da1da..7e5229e 100644 --- a/jinja2/nodes.py +++ b/jinja2/nodes.py @@ -13,9 +13,10 @@ :license: BSD, see LICENSE for more details. """ import operator -from itertools import chain, izip from collections import deque from jinja2.utils import Markup, MethodType, FunctionType +import six +from six.moves import zip #: the types we support for context functions @@ -102,9 +103,9 @@ def get_eval_context(node, ctx): return ctx -class Node(object): +class Node(six.with_metaclass(NodeType, object)): """Baseclass for all Jinja2 nodes. There are a number of nodes available - of different types. There are three major types: + of different types. There are four major types: - :class:`Stmt`: statements - :class:`Expr`: expressions @@ -136,13 +137,13 @@ class Node(object): len(self.fields), len(self.fields) != 1 and 's' or '' )) - for name, arg in izip(self.fields, fields): + for name, arg in zip(self.fields, fields): setattr(self, name, arg) for attr in self.attributes: setattr(self, attr, attributes.pop(attr, None)) if attributes: raise TypeError('unknown attribute %r' % - iter(attributes).next()) + six.advance_iterator(iter(attributes))) def iter_fields(self, exclude=None, only=None): """This method iterates over all fields that are defined and yields @@ -440,7 +441,7 @@ class Const(Literal): constant value in the generated code, otherwise it will raise an `Impossible` exception. """ - from compiler import has_safe_repr + from .compiler import has_safe_repr if not has_safe_repr(value): raise Impossible() return cls(value, lineno=lineno, environment=environment) @@ -687,7 +688,7 @@ class Concat(Expr): def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) - return ''.join(unicode(x.as_const(eval_ctx)) for x in self.nodes) + return ''.join(six.text_type(x.as_const(eval_ctx)) for x in self.nodes) class Compare(Expr): diff --git a/jinja2/parser.py b/jinja2/parser.py index 2125338..953027c 100644 --- a/jinja2/parser.py +++ b/jinja2/parser.py @@ -10,8 +10,9 @@ """ from jinja2 import nodes from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError -from jinja2.utils import next from jinja2.lexer import describe_token, describe_token_expr +import six +from six.moves import map #: statements that callinto @@ -162,12 +163,12 @@ class Parser(object): self.fail_eof(end_tokens) if drop_needle: - next(self.stream) + six.advance_iterator(self.stream) return result def parse_set(self): """Parse an assign statement.""" - lineno = next(self.stream).lineno + lineno = six.advance_iterator(self.stream).lineno target = self.parse_assign_target() self.stream.expect('assign') expr = self.parse_tuple() @@ -185,7 +186,7 @@ class Parser(object): test = self.parse_expression() recursive = self.stream.skip_if('name:recursive') body = self.parse_statements(('name:endfor', 'name:else')) - if next(self.stream).value == 'endfor': + if six.advance_iterator(self.stream).value == 'endfor': else_ = [] else: else_ = self.parse_statements(('name:endfor',), drop_needle=True) @@ -199,7 +200,7 @@ class Parser(object): node.test = self.parse_tuple(with_condexpr=False) node.body = self.parse_statements(('name:elif', 'name:else', 'name:endif')) - token = next(self.stream) + token = six.advance_iterator(self.stream) if token.test('name:elif'): new_node = nodes.If(lineno=self.stream.current.lineno) node.else_ = [new_node] @@ -214,7 +215,7 @@ class Parser(object): return result def parse_block(self): - node = nodes.Block(lineno=next(self.stream).lineno) + node = nodes.Block(lineno=six.advance_iterator(self.stream).lineno) node.name = self.stream.expect('name').value node.scoped = self.stream.skip_if('name:scoped') @@ -231,21 +232,21 @@ class Parser(object): return node def parse_extends(self): - node = nodes.Extends(lineno=next(self.stream).lineno) + node = nodes.Extends(lineno=six.advance_iterator(self.stream).lineno) node.template = self.parse_expression() return node def parse_import_context(self, node, default): if self.stream.current.test_any('name:with', 'name:without') and \ self.stream.look().test('name:context'): - node.with_context = next(self.stream).value == 'with' + node.with_context = six.advance_iterator(self.stream).value == 'with' self.stream.skip() else: node.with_context = default return node def parse_include(self): - node = nodes.Include(lineno=next(self.stream).lineno) + node = nodes.Include(lineno=six.advance_iterator(self.stream).lineno) node.template = self.parse_expression() if self.stream.current.test('name:ignore') and \ self.stream.look().test('name:missing'): @@ -256,14 +257,14 @@ class Parser(object): return self.parse_import_context(node, True) def parse_import(self): - node = nodes.Import(lineno=next(self.stream).lineno) + node = nodes.Import(lineno=six.advance_iterator(self.stream).lineno) node.template = self.parse_expression() self.stream.expect('name:as') node.target = self.parse_assign_target(name_only=True).name return self.parse_import_context(node, False) def parse_from(self): - node = nodes.FromImport(lineno=next(self.stream).lineno) + node = nodes.FromImport(lineno=six.advance_iterator(self.stream).lineno) node.template = self.parse_expression() self.stream.expect('name:import') node.names = [] @@ -271,7 +272,7 @@ class Parser(object): def parse_context(): if self.stream.current.value in ('with', 'without') and \ self.stream.look().test('name:context'): - node.with_context = next(self.stream).value == 'with' + node.with_context = six.advance_iterator(self.stream).value == 'with' self.stream.skip() return True return False @@ -316,7 +317,7 @@ class Parser(object): self.stream.expect('rparen') def parse_call_block(self): - node = nodes.CallBlock(lineno=next(self.stream).lineno) + node = nodes.CallBlock(lineno=six.advance_iterator(self.stream).lineno) if self.stream.current.type == 'lparen': self.parse_signature(node) else: @@ -330,14 +331,14 @@ class Parser(object): return node def parse_filter_block(self): - node = nodes.FilterBlock(lineno=next(self.stream).lineno) + node = nodes.FilterBlock(lineno=six.advance_iterator(self.stream).lineno) node.filter = self.parse_filter(None, start_inline=True) node.body = self.parse_statements(('name:endfilter',), drop_needle=True) return node def parse_macro(self): - node = nodes.Macro(lineno=next(self.stream).lineno) + node = nodes.Macro(lineno=six.advance_iterator(self.stream).lineno) node.name = self.parse_assign_target(name_only=True).name self.parse_signature(node) node.body = self.parse_statements(('name:endmacro',), @@ -345,7 +346,7 @@ class Parser(object): return node def parse_print(self): - node = nodes.Output(lineno=next(self.stream).lineno) + node = nodes.Output(lineno=six.advance_iterator(self.stream).lineno) node.nodes = [] while self.stream.current.type != 'block_end': if node.nodes: @@ -419,7 +420,7 @@ class Parser(object): def parse_not(self): if self.stream.current.test('name:not'): - lineno = next(self.stream).lineno + lineno = six.advance_iterator(self.stream).lineno return nodes.Not(self.parse_not(), lineno=lineno) return self.parse_compare() @@ -430,7 +431,7 @@ class Parser(object): while 1: token_type = self.stream.current.type if token_type in _compare_operators: - next(self.stream) + six.advance_iterator(self.stream) ops.append(nodes.Operand(token_type, self.parse_add())) elif self.stream.skip_if('name:in'): ops.append(nodes.Operand('in', self.parse_add())) @@ -449,7 +450,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_sub() while self.stream.current.type == 'add': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_sub() left = nodes.Add(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -459,7 +460,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_concat() while self.stream.current.type == 'sub': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_concat() left = nodes.Sub(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -469,7 +470,7 @@ class Parser(object): lineno = self.stream.current.lineno args = [self.parse_mul()] while self.stream.current.type == 'tilde': - next(self.stream) + six.advance_iterator(self.stream) args.append(self.parse_mul()) if len(args) == 1: return args[0] @@ -479,7 +480,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_div() while self.stream.current.type == 'mul': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_div() left = nodes.Mul(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -489,7 +490,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_floordiv() while self.stream.current.type == 'div': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_floordiv() left = nodes.Div(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -499,7 +500,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_mod() while self.stream.current.type == 'floordiv': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_mod() left = nodes.FloorDiv(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -509,7 +510,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_pow() while self.stream.current.type == 'mod': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_pow() left = nodes.Mod(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -519,7 +520,7 @@ class Parser(object): lineno = self.stream.current.lineno left = self.parse_unary() while self.stream.current.type == 'pow': - next(self.stream) + six.advance_iterator(self.stream) right = self.parse_unary() left = nodes.Pow(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -529,10 +530,10 @@ class Parser(object): token_type = self.stream.current.type lineno = self.stream.current.lineno if token_type == 'sub': - next(self.stream) + six.advance_iterator(self.stream) node = nodes.Neg(self.parse_unary(False), lineno=lineno) elif token_type == 'add': - next(self.stream) + six.advance_iterator(self.stream) node = nodes.Pos(self.parse_unary(False), lineno=lineno) else: node = self.parse_primary() @@ -551,20 +552,20 @@ class Parser(object): node = nodes.Const(None, lineno=token.lineno) else: node = nodes.Name(token.value, 'load', lineno=token.lineno) - next(self.stream) + six.advance_iterator(self.stream) elif token.type == 'string': - next(self.stream) + six.advance_iterator(self.stream) buf = [token.value] lineno = token.lineno while self.stream.current.type == 'string': buf.append(self.stream.current.value) - next(self.stream) + six.advance_iterator(self.stream) node = nodes.Const(''.join(buf), lineno=lineno) elif token.type in ('integer', 'float'): - next(self.stream) + six.advance_iterator(self.stream) node = nodes.Const(token.value, lineno=token.lineno) elif token.type == 'lparen': - next(self.stream) + six.advance_iterator(self.stream) node = self.parse_tuple(explicit_parentheses=True) self.stream.expect('rparen') elif token.type == 'lbracket': @@ -686,10 +687,10 @@ class Parser(object): return node def parse_subscript(self, node): - token = next(self.stream) + token = six.advance_iterator(self.stream) if token.type == 'dot': attr_token = self.stream.current - next(self.stream) + six.advance_iterator(self.stream) if attr_token.type == 'name': return nodes.Getattr(node, attr_token.value, 'load', lineno=token.lineno) @@ -715,13 +716,13 @@ class Parser(object): lineno = self.stream.current.lineno if self.stream.current.type == 'colon': - next(self.stream) + six.advance_iterator(self.stream) args = [None] else: node = self.parse_expression() if self.stream.current.type != 'colon': return node - next(self.stream) + six.advance_iterator(self.stream) args = [node] if self.stream.current.type == 'colon': @@ -732,7 +733,7 @@ class Parser(object): args.append(None) if self.stream.current.type == 'colon': - next(self.stream) + six.advance_iterator(self.stream) if self.stream.current.type not in ('rbracket', 'comma'): args.append(self.parse_expression()) else: @@ -762,11 +763,11 @@ class Parser(object): break if self.stream.current.type == 'mul': ensure(dyn_args is None and dyn_kwargs is None) - next(self.stream) + six.advance_iterator(self.stream) dyn_args = self.parse_expression() elif self.stream.current.type == 'pow': ensure(dyn_kwargs is None) - next(self.stream) + six.advance_iterator(self.stream) dyn_kwargs = self.parse_expression() else: ensure(dyn_args is None and dyn_kwargs is None) @@ -792,11 +793,11 @@ class Parser(object): def parse_filter(self, node, start_inline=False): while self.stream.current.type == 'pipe' or start_inline: if not start_inline: - next(self.stream) + six.advance_iterator(self.stream) token = self.stream.expect('name') name = token.value while self.stream.current.type == 'dot': - next(self.stream) + six.advance_iterator(self.stream) name += '.' + self.stream.expect('name').value if self.stream.current.type == 'lparen': args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) @@ -810,15 +811,15 @@ class Parser(object): return node def parse_test(self, node): - token = next(self.stream) + token = six.advance_iterator(self.stream) if self.stream.current.test('name:not'): - next(self.stream) + six.advance_iterator(self.stream) negated = True else: negated = False name = self.stream.expect('name').value while self.stream.current.type == 'dot': - next(self.stream) + six.advance_iterator(self.stream) name += '.' + self.stream.expect('name').value dyn_args = dyn_kwargs = None kwargs = [] @@ -861,14 +862,14 @@ class Parser(object): if token.value: add_data(nodes.TemplateData(token.value, lineno=token.lineno)) - next(self.stream) + six.advance_iterator(self.stream) elif token.type == 'variable_begin': - next(self.stream) + six.advance_iterator(self.stream) add_data(self.parse_tuple(with_condexpr=True)) self.stream.expect('variable_end') elif token.type == 'block_begin': flush_data() - next(self.stream) + six.advance_iterator(self.stream) if end_tokens is not None and \ self.stream.current.test_any(*end_tokens): return body diff --git a/jinja2/runtime.py b/jinja2/runtime.py index a4a47a2..9742ece 100644 --- a/jinja2/runtime.py +++ b/jinja2/runtime.py @@ -8,12 +8,14 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD. """ -from itertools import chain, imap +from itertools import chain from jinja2.nodes import EvalContext, _context_function_types -from jinja2.utils import Markup, partial, soft_unicode, escape, missing, \ - concat, internalcode, next, object_type_repr +from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \ + internalcode, object_type_repr from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \ TemplateNotFound +import six +from six.moves import map # these variables are exported to the template runtime @@ -25,16 +27,21 @@ __all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup', #: the name of the function that is used to convert something into #: a string. 2to3 will adopt that automatically and the generated #: code can take advantage of it. -to_string = unicode +try: + to_string = unicode +except NameError: + to_string = str #: the identity function. Useful for certain things in the environment identity = lambda x: x +_last_iteration = object() + def markup_join(seq): """Concatenation that escapes if necessary and converts to unicode.""" buf = [] - iterator = imap(soft_unicode, seq) + iterator = map(soft_unicode, seq) for arg in iterator: buf.append(arg) if hasattr(arg, '__html__'): @@ -44,7 +51,7 @@ def markup_join(seq): def unicode_join(seq): """Simple args to unicode conversion and concatenation.""" - return concat(imap(unicode, seq)) + return concat(map(six.text_type, seq)) def new_context(environment, template_name, blocks, vars=None, @@ -61,7 +68,7 @@ def new_context(environment, template_name, blocks, vars=None, # we don't want to modify the dict passed if shared: parent = dict(parent) - for key, value in locals.iteritems(): + for key, value in six.iteritems(locals): if key[:2] == 'l_' and value is not missing: parent[key[2:]] = value return Context(environment, parent, template_name, blocks) @@ -117,7 +124,7 @@ class Context(object): # create the initial mapping of blocks. Whenever template inheritance # takes place the runtime will update this mapping with the new blocks # from the template. - self.blocks = dict((k, [v]) for k, v in blocks.iteritems()) + self.blocks = dict((k, [v]) for k, v in six.iteritems(blocks)) def super(self, name, current): """Render a parent block.""" @@ -169,6 +176,17 @@ class Context(object): """ if __debug__: __traceback_hide__ = True + + # Allow callable classes to take a context + if hasattr(__obj, '__call__'): + fn = __obj.__call__ + for fn_type in ('contextfunction', + 'evalcontextfunction', + 'environmentfunction'): + if hasattr(fn, fn_type): + __obj = fn + break + if isinstance(__obj, _context_function_types): if getattr(__obj, 'contextfunction', 0): args = (__self,) + args @@ -189,7 +207,7 @@ class Context(object): self.parent, True, None, locals) context.vars.update(self.vars) context.eval_ctx = self.eval_ctx - context.blocks.update((k, list(v)) for k, v in self.blocks.iteritems()) + context.blocks.update((k, list(v)) for k, v in six.iteritems(self.blocks)) return context def _all(meth): @@ -270,6 +288,7 @@ class LoopContext(object): def __init__(self, iterable, recurse=None): self._iterator = iter(iterable) self._recurse = recurse + self._after = self._safe_next() self.index0 = -1 # try to get the length of the iterable early. This must be done @@ -288,7 +307,7 @@ class LoopContext(object): return args[self.index0 % len(args)] first = property(lambda x: x.index0 == 0) - last = property(lambda x: x.index0 + 1 == x.length) + last = property(lambda x: x._after is _last_iteration) index = property(lambda x: x.index0 + 1) revindex = property(lambda x: x.length - x.index0) revindex0 = property(lambda x: x.length - x.index) @@ -299,6 +318,12 @@ class LoopContext(object): def __iter__(self): return LoopContextIterator(self) + def _safe_next(self): + try: + return six.advance_iterator(self._iterator) + except StopIteration: + return _last_iteration + @internalcode def loop(self, iterable): if self._recurse is None: @@ -331,7 +356,7 @@ class LoopContext(object): ) -class LoopContextIterator(object): +class LoopContextIterator(six.Iterator): """The iterator for a loop context.""" __slots__ = ('context',) @@ -341,10 +366,14 @@ class LoopContextIterator(object): def __iter__(self): return self - def next(self): + def __next__(self): ctx = self.context ctx.index0 += 1 - return next(ctx._iterator), ctx + if ctx._after is _last_iteration: + raise StopIteration() + next_elem = ctx._after + ctx._after = ctx._safe_next() + return next_elem, ctx class Macro(object): @@ -442,7 +471,7 @@ class Undefined(object): if self._undefined_hint is None: if self._undefined_obj is missing: hint = '%r is undefined' % self._undefined_name - elif not isinstance(self._undefined_name, basestring): + elif not isinstance(self._undefined_name, six.string_types): hint = '%s has no element %r' % ( object_type_repr(self._undefined_obj), self._undefined_name @@ -470,12 +499,9 @@ class Undefined(object): _fail_with_undefined_error def __str__(self): - return unicode(self).encode('utf-8') + s = self.__unicode__() + return s if six.PY3 else s.encode('utf-8') - # unicode goes after __str__ because we configured 2to3 to rename - # __unicode__ to __str__. because the 2to3 tree is not designed to - # remove nodes from it, we leave the above __str__ around and let - # it override at runtime. def __unicode__(self): return u'' @@ -508,6 +534,10 @@ class DebugUndefined(Undefined): """ __slots__ = () + def __str__(self): + s = self.__unicode__() + return s if six.PY3 else s.encode('utf-8') + def __unicode__(self): if self._undefined_hint is None: if self._undefined_obj is missing: diff --git a/jinja2/sandbox.py b/jinja2/sandbox.py index a1cbb29..ed145d5 100644 --- a/jinja2/sandbox.py +++ b/jinja2/sandbox.py @@ -13,6 +13,7 @@ :license: BSD. """ import operator +import six from jinja2.environment import Environment from jinja2.exceptions import SecurityError from jinja2.utils import FunctionType, MethodType, TracebackType, CodeType, \ @@ -90,7 +91,7 @@ def safe_range(*args): """A range that can't generate ranges with a length of more than MAX_RANGE items. """ - rng = xrange(*args) + rng = range(*args) if len(rng) > MAX_RANGE: raise OverflowError('range too big, maximum size for range is %d' % MAX_RANGE) @@ -114,7 +115,7 @@ def is_internal_attribute(obj, attr): """Test if the attribute given is an internal python attribute. For example this function returns `True` for the `func_code` attribute of python objects. This is useful if the environment method - :meth:`~SandboxedEnvironment.is_safe_attribute` is overriden. + :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden. >>> from jinja2.sandbox import is_internal_attribute >>> is_internal_attribute(lambda: None, "func_code") @@ -299,7 +300,7 @@ class SandboxedEnvironment(Environment): try: return obj[argument] except (TypeError, LookupError): - if isinstance(argument, basestring): + if isinstance(argument, six.string_types): try: attr = str(argument) except Exception: diff --git a/jinja2/tests.py b/jinja2/tests.py index 50510b0..140b5bf 100644 --- a/jinja2/tests.py +++ b/jinja2/tests.py @@ -10,6 +10,7 @@ """ import re from jinja2.runtime import Undefined +import six try: from collections import Mapping as MappingType @@ -76,17 +77,17 @@ def test_none(value): def test_lower(value): """Return true if the variable is lowercased.""" - return unicode(value).islower() + return six.text_type(value).islower() def test_upper(value): """Return true if the variable is uppercased.""" - return unicode(value).isupper() + return six.text_type(value).isupper() def test_string(value): """Return true if the object is a string.""" - return isinstance(value, basestring) + return isinstance(value, six.string_types) def test_mapping(value): @@ -99,7 +100,7 @@ def test_mapping(value): def test_number(value): """Return true if the variable is a number.""" - return isinstance(value, (int, long, float, complex)) + return isinstance(value, (int, float, complex)) def test_sequence(value): diff --git a/jinja2/testsuite/__init__.py b/jinja2/testsuite/__init__.py index 1f10ef6..335292d 100644 --- a/jinja2/testsuite/__init__.py +++ b/jinja2/testsuite/__init__.py @@ -59,7 +59,7 @@ class JinjaTestCase(unittest.TestCase): def assert_traceback_matches(self, callback, expected_tb): try: callback() - except Exception, e: + except Exception as e: tb = format_exception(*sys.exc_info()) if re.search(expected_tb.strip(), ''.join(tb)) is None: raise self.fail('Traceback did not match:\n\n%s\nexpected:\n%s' @@ -89,7 +89,7 @@ def suite(): # doctests will not run on python 3 currently. Too many issues # with that, do not test that on that platform. - if sys.version_info < (3, 0): + if sys.version_info[0] < 3: suite.addTest(doctests.suite()) return suite diff --git a/jinja2/testsuite/api.py b/jinja2/testsuite/api.py index c8f9634..f6bca48 100644 --- a/jinja2/testsuite/api.py +++ b/jinja2/testsuite/api.py @@ -9,6 +9,9 @@ :license: BSD, see LICENSE for more details. """ import unittest +import os +import tempfile +import shutil from jinja2.testsuite import JinjaTestCase @@ -16,6 +19,7 @@ from jinja2 import Environment, Undefined, DebugUndefined, \ StrictUndefined, UndefinedError, meta, \ is_undefined, Template, DictLoader from jinja2.utils import Cycler +import six env = Environment() @@ -50,8 +54,8 @@ class ExtendedAPITestCase(JinjaTestCase): c = Cycler(*items) for item in items + items: assert c.current == item - assert c.next() == item - c.next() + assert six.advance_iterator(c) == item + six.advance_iterator(c) assert c.current == 2 c.reset() assert c.current == 1 @@ -107,8 +111,8 @@ class MetaTestCase(JinjaTestCase): def test_find_refererenced_templates(self): ast = env.parse('{% extends "layout.html" %}{% include helper %}') i = meta.find_referenced_templates(ast) - assert i.next() == 'layout.html' - assert i.next() is None + assert six.advance_iterator(i) == 'layout.html' + assert six.advance_iterator(i) is None assert list(i) == [] ast = env.parse('{% extends "layout.html" %}' @@ -141,21 +145,21 @@ class StreamingTestCase(JinjaTestCase): def test_basic_streaming(self): tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index " "}} - {{ item }}</li>{%- endfor %}</ul>") - stream = tmpl.stream(seq=range(4)) - self.assert_equal(stream.next(), '<ul>') - self.assert_equal(stream.next(), '<li>1 - 0</li>') - self.assert_equal(stream.next(), '<li>2 - 1</li>') - self.assert_equal(stream.next(), '<li>3 - 2</li>') - self.assert_equal(stream.next(), '<li>4 - 3</li>') - self.assert_equal(stream.next(), '</ul>') + stream = tmpl.stream(seq=list(range(4))) + self.assert_equal(six.advance_iterator(stream), '<ul>') + self.assert_equal(six.advance_iterator(stream), '<li>1 - 0</li>') + self.assert_equal(six.advance_iterator(stream), '<li>2 - 1</li>') + self.assert_equal(six.advance_iterator(stream), '<li>3 - 2</li>') + self.assert_equal(six.advance_iterator(stream), '<li>4 - 3</li>') + self.assert_equal(six.advance_iterator(stream), '</ul>') def test_buffered_streaming(self): tmpl = env.from_string("<ul>{% for item in seq %}<li>{{ loop.index " "}} - {{ item }}</li>{%- endfor %}</ul>") - stream = tmpl.stream(seq=range(4)) + stream = tmpl.stream(seq=list(range(4))) stream.enable_buffering(size=3) - self.assert_equal(stream.next(), u'<ul><li>1 - 0</li><li>2 - 1</li>') - self.assert_equal(stream.next(), u'<li>3 - 2</li><li>4 - 3</li></ul>') + self.assert_equal(six.advance_iterator(stream), u'<ul><li>1 - 0</li><li>2 - 1</li>') + self.assert_equal(six.advance_iterator(stream), u'<li>3 - 2</li><li>4 - 3</li></ul>') def test_streaming_behavior(self): tmpl = env.from_string("") @@ -166,6 +170,17 @@ class StreamingTestCase(JinjaTestCase): stream.disable_buffering() assert not stream.buffered + def test_dump_stream(self): + tmp = tempfile.mkdtemp() + try: + tmpl = env.from_string(u"\u2713") + stream = tmpl.stream() + stream.dump(os.path.join(tmp, 'dump.txt'), 'utf-8') + with open(os.path.join(tmp, 'dump.txt'), 'rb') as f: + self.assertEqual(f.read(), b'\xe2\x9c\x93') + finally: + shutil.rmtree(tmp) + class UndefinedTestCase(JinjaTestCase): @@ -214,6 +229,7 @@ class UndefinedTestCase(JinjaTestCase): self.assert_equal(env.from_string('{{ missing is not defined }}').render(), 'True') self.assert_raises(UndefinedError, env.from_string('{{ foo.missing }}').render, foo=42) self.assert_raises(UndefinedError, env.from_string('{{ not missing }}').render) + self.assert_equal(env.from_string('{{ missing|default("default", true) }}').render(), 'default') def test_indexing_gives_undefined(self): t = Template("{{ var[42].foo }}") @@ -222,7 +238,7 @@ class UndefinedTestCase(JinjaTestCase): def test_none_gives_proper_error(self): try: Environment().getattr(None, 'split')() - except UndefinedError, e: + except UndefinedError as e: assert e.message == "'None' has no attribute 'split'" else: assert False, 'expected exception' @@ -230,7 +246,7 @@ class UndefinedTestCase(JinjaTestCase): def test_object_repr(self): try: Undefined(obj=42, name='upper')() - except UndefinedError, e: + except UndefinedError as e: assert e.message == "'int object' has no attribute 'upper'" else: assert False, 'expected exception' diff --git a/jinja2/testsuite/core_tags.py b/jinja2/testsuite/core_tags.py index 2b5f580..409a579 100644 --- a/jinja2/testsuite/core_tags.py +++ b/jinja2/testsuite/core_tags.py @@ -22,7 +22,7 @@ class ForLoopTestCase(JinjaTestCase): def test_simple(self): tmpl = env.from_string('{% for item in seq %}{{ item }}{% endfor %}') - assert tmpl.render(seq=range(10)) == '0123456789' + assert tmpl.render(seq=list(range(10))) == '0123456789' def test_else(self): tmpl = env.from_string('{% for item in seq %}XXX{% else %}...{% endfor %}') @@ -55,12 +55,12 @@ class ForLoopTestCase(JinjaTestCase): tmpl = env.from_string('''{% for item in seq %}{{ loop.cycle('<1>', '<2>') }}{% endfor %}{% for item in seq %}{{ loop.cycle(*through) }}{% endfor %}''') - output = tmpl.render(seq=range(4), through=('<1>', '<2>')) + output = tmpl.render(seq=list(range(4)), through=('<1>', '<2>')) assert output == '<1><2>' * 4 def test_scope(self): tmpl = env.from_string('{% for item in seq %}{% endfor %}{{ item }}') - output = tmpl.render(seq=range(10)) + output = tmpl.render(seq=list(range(10))) assert not output def test_varlen(self): diff --git a/jinja2/testsuite/debug.py b/jinja2/testsuite/debug.py index 7552dec..2588a83 100644 --- a/jinja2/testsuite/debug.py +++ b/jinja2/testsuite/debug.py @@ -8,7 +8,6 @@ :copyright: (c) 2010 by the Jinja Team. :license: BSD, see LICENSE for more details. """ -import sys import unittest from jinja2.testsuite import JinjaTestCase, filesystem_loader @@ -20,12 +19,11 @@ env = Environment(loader=filesystem_loader) class DebugTestCase(JinjaTestCase): - if sys.version_info[:2] != (2, 4): - def test_runtime_error(self): - def test(): - tmpl.render(fail=lambda: 1 / 0) - tmpl = env.get_template('broken.html') - self.assert_traceback_matches(test, r''' + def test_runtime_error(self): + def test(): + tmpl.render(fail=lambda: 1 / 0) + tmpl = env.get_template('broken.html') + self.assert_traceback_matches(test, r''' File ".*?broken.html", line 2, in (top-level template code|<module>) \{\{ fail\(\) \}\} File ".*?debug.pyc?", line \d+, in <lambda> diff --git a/jinja2/testsuite/ext.py b/jinja2/testsuite/ext.py index 2a16701..817b1d2 100644 --- a/jinja2/testsuite/ext.py +++ b/jinja2/testsuite/ext.py @@ -17,14 +17,8 @@ from jinja2 import Environment, DictLoader, contextfunction, nodes from jinja2.exceptions import TemplateAssertionError from jinja2.ext import Extension from jinja2.lexer import Token, count_newlines -from jinja2.utils import next - -# 2.x / 3.x -try: - from io import BytesIO -except ImportError: - from StringIO import StringIO as BytesIO - +import six +from six import BytesIO importable_object = 23 @@ -117,7 +111,7 @@ class TestExtension(Extension): self.attr('ext_attr'), nodes.ImportedName(__name__ + '.importable_object'), nodes.ContextReference() - ])]).set_lineno(next(parser.stream).lineno) + ])]).set_lineno(six.advance_iterator(parser.stream).lineno) def _dump(self, sandboxed, ext_attr, imported_object, context): return '%s|%s|%s|%s' % ( @@ -224,7 +218,7 @@ class ExtensionsTestCase(JinjaTestCase): original = Environment(extensions=[TestExtension]) overlay = original.overlay() for env in original, overlay: - for ext in env.extensions.itervalues(): + for ext in six.itervalues(env.extensions): assert ext.environment is env def test_preprocessor_extension(self): @@ -443,7 +437,7 @@ class AutoEscapeTestCase(JinjaTestCase): ''' tmpl = env.from_string(tmplsource) assert tmpl.render(val=True).split()[0] == 'Markup' - assert tmpl.render(val=False).split()[0] == unicode.__name__ + assert tmpl.render(val=False).split()[0] == six.text_type.__name__ # looking at the source we should see <testing> there in raw # (and then escaped as well) diff --git a/jinja2/testsuite/filters.py b/jinja2/testsuite/filters.py index 94bc02a..88f93f9 100644 --- a/jinja2/testsuite/filters.py +++ b/jinja2/testsuite/filters.py @@ -12,6 +12,8 @@ import unittest from jinja2.testsuite import JinjaTestCase from jinja2 import Markup, Environment +import six +from six.moves import map env = Environment() @@ -47,14 +49,14 @@ class FilterTestCase(JinjaTestCase): def test_batch(self): tmpl = env.from_string("{{ foo|batch(3)|list }}|" "{{ foo|batch(3, 'X')|list }}") - out = tmpl.render(foo=range(10)) + out = tmpl.render(foo=list(range(10))) assert out == ("[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]|" "[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 'X', 'X']]") def test_slice(self): tmpl = env.from_string('{{ foo|slice(3)|list }}|' '{{ foo|slice(3, "X")|list }}') - out = tmpl.render(foo=range(10)) + out = tmpl.render(foo=list(range(10))) assert out == ("[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|" "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]") @@ -109,7 +111,7 @@ class FilterTestCase(JinjaTestCase): def test_first(self): tmpl = env.from_string('{{ foo|first }}') - out = tmpl.render(foo=range(10)) + out = tmpl.render(foo=list(range(10))) assert out == '0' def test_float(self): @@ -155,7 +157,7 @@ class FilterTestCase(JinjaTestCase): def test_last(self): tmpl = env.from_string('''{{ foo|last }}''') - out = tmpl.render(foo=range(10)) + out = tmpl.render(foo=list(range(10))) assert out == '9' def test_length(self): @@ -171,12 +173,12 @@ class FilterTestCase(JinjaTestCase): def test_pprint(self): from pprint import pformat tmpl = env.from_string('''{{ data|pprint }}''') - data = range(1000) + data = list(range(1000)) assert tmpl.render(data=data) == pformat(data) def test_random(self): tmpl = env.from_string('''{{ seq|random }}''') - seq = range(100) + seq = list(range(100)) for _ in range(10): assert int(tmpl.render(seq=seq)) in seq @@ -188,11 +190,21 @@ class FilterTestCase(JinjaTestCase): def test_string(self): x = [1, 2, 3, 4, 5] tmpl = env.from_string('''{{ obj|string }}''') - assert tmpl.render(obj=x) == unicode(x) + assert tmpl.render(obj=x) == six.text_type(x) def test_title(self): tmpl = env.from_string('''{{ "foo bar"|title }}''') assert tmpl.render() == "Foo Bar" + tmpl = env.from_string('''{{ "foo's bar"|title }}''') + assert tmpl.render() == "Foo's Bar" + tmpl = env.from_string('''{{ "foo bar"|title }}''') + assert tmpl.render() == "Foo Bar" + tmpl = env.from_string('''{{ "f bar f"|title }}''') + assert tmpl.render() == "F Bar F" + tmpl = env.from_string('''{{ "foo-bar"|title }}''') + assert tmpl.render() == "Foo-Bar" + tmpl = env.from_string('''{{ "foo\tbar"|title }}''') + assert tmpl.render() == "Foo\tBar" def test_truncate(self): tmpl = env.from_string( @@ -287,7 +299,8 @@ class FilterTestCase(JinjaTestCase): def __init__(self, value): self.value = value def __unicode__(self): - return unicode(self.value) + return six.text_type(self.value) + __str__ = __unicode__ tmpl = env.from_string('''{{ items|sort(attribute='value')|join }}''') assert tmpl.render(items=map(Magic, [3, 2, 4, 1])) == '1234' @@ -379,6 +392,109 @@ class FilterTestCase(JinjaTestCase): assert tmpl.render(o={u"\u203d": 1}) == "%E2%80%BD=1" assert tmpl.render(o={0: 1}) == "0=1" + def test_simple_map(self): + env = Environment() + tmpl = env.from_string('{{ ["1", "2", "3"]|map("int")|sum }}') + self.assertEqual(tmpl.render(), '6') + + def test_attribute_map(self): + class User(object): + def __init__(self, name): + self.name = name + env = Environment() + users = [ + User('john'), + User('jane'), + User('mike'), + ] + tmpl = env.from_string('{{ users|map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'john|jane|mike') + + def test_empty_map(self): + env = Environment() + tmpl = env.from_string('{{ none|map("upper")|list }}') + self.assertEqual(tmpl.render(), '[]') + + def test_simple_select(self): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|select("odd")|join("|") }}') + self.assertEqual(tmpl.render(), '1|3|5') + + def test_bool_select(self): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|select|join("|") }}') + self.assertEqual(tmpl.render(), '1|2|3|4|5') + + def test_simple_reject(self): + env = Environment() + tmpl = env.from_string('{{ [1, 2, 3, 4, 5]|reject("odd")|join("|") }}') + self.assertEqual(tmpl.render(), '2|4') + + def test_bool_reject(self): + env = Environment() + tmpl = env.from_string('{{ [none, false, 0, 1, 2, 3, 4, 5]|reject|join("|") }}') + self.assertEqual(tmpl.render(), 'None|False|0') + + def test_simple_select_attr(self): + class User(object): + def __init__(self, name, is_active): + self.name = name + self.is_active = is_active + env = Environment() + users = [ + User('john', True), + User('jane', True), + User('mike', False), + ] + tmpl = env.from_string('{{ users|selectattr("is_active")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'john|jane') + + def test_simple_reject_attr(self): + class User(object): + def __init__(self, name, is_active): + self.name = name + self.is_active = is_active + env = Environment() + users = [ + User('john', True), + User('jane', True), + User('mike', False), + ] + tmpl = env.from_string('{{ users|rejectattr("is_active")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'mike') + + def test_func_select_attr(self): + class User(object): + def __init__(self, id, name): + self.id = id + self.name = name + env = Environment() + users = [ + User(1, 'john'), + User(2, 'jane'), + User(3, 'mike'), + ] + tmpl = env.from_string('{{ users|selectattr("id", "odd")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'john|mike') + + def test_func_reject_attr(self): + class User(object): + def __init__(self, id, name): + self.id = id + self.name = name + env = Environment() + users = [ + User(1, 'john'), + User(2, 'jane'), + User(3, 'mike'), + ] + tmpl = env.from_string('{{ users|rejectattr("id", "odd")|' + 'map(attribute="name")|join("|") }}') + self.assertEqual(tmpl.render(users=users), 'jane') + def suite(): suite = unittest.TestSuite() diff --git a/jinja2/testsuite/imports.py b/jinja2/testsuite/imports.py index 1cb12cb..3db9008 100644 --- a/jinja2/testsuite/imports.py +++ b/jinja2/testsuite/imports.py @@ -83,7 +83,7 @@ class IncludesTestCase(JinjaTestCase): self.assert_raises(TemplateNotFound, t.render) try: t.render() - except TemplatesNotFound, e: + except TemplatesNotFound as e: assert e.templates == ['missing', 'missing2'] assert e.name == 'missing2' else: diff --git a/jinja2/testsuite/inheritance.py b/jinja2/testsuite/inheritance.py index 355aa0c..7909b03 100644 --- a/jinja2/testsuite/inheritance.py +++ b/jinja2/testsuite/inheritance.py @@ -148,7 +148,7 @@ class InheritanceTestCase(JinjaTestCase): })) t = env.from_string('{% extends "master.html" %}{% block item %}' '{{ item }}{% endblock %}') - assert t.render(seq=range(5)) == '[0][1][2][3][4]' + assert t.render(seq=list(range(5))) == '[0][1][2][3][4]' def test_super_in_scoped_block(self): env = Environment(loader=DictLoader({ @@ -157,7 +157,7 @@ class InheritanceTestCase(JinjaTestCase): })) t = env.from_string('{% extends "master.html" %}{% block item %}' '{{ super() }}|{{ item * 2 }}{% endblock %}') - assert t.render(seq=range(5)) == '[0|0][1|2][2|4][3|6][4|8]' + assert t.render(seq=list(range(5))) == '[0|0][1|2][2|4][3|6][4|8]' def test_scoped_block_after_inheritance(self): env = Environment(loader=DictLoader({ diff --git a/jinja2/testsuite/lexnparse.py b/jinja2/testsuite/lexnparse.py index 77b76ec..d2473cf 100644 --- a/jinja2/testsuite/lexnparse.py +++ b/jinja2/testsuite/lexnparse.py @@ -15,18 +15,44 @@ from jinja2.testsuite import JinjaTestCase from jinja2 import Environment, Template, TemplateSyntaxError, \ UndefinedError, nodes +from jinja2.lexer import Token, TokenStream, TOKEN_EOF, TOKEN_BLOCK_BEGIN, TOKEN_BLOCK_END +import six env = Environment() # how does a string look like in jinja syntax? -if sys.version_info < (3, 0): +if sys.version_info[0] < 3: def jinja_string_repr(string): return repr(string)[1:] else: jinja_string_repr = repr +class TokenStreamTestCase(JinjaTestCase): + test_tokens = [Token(1, TOKEN_BLOCK_BEGIN, ''), + Token(2, TOKEN_BLOCK_END, ''), + ] + + def test_simple(self): + ts = TokenStream(self.test_tokens, "foo", "bar") + assert ts.current.type is TOKEN_BLOCK_BEGIN + assert bool(ts) + assert not bool(ts.eos) + six.advance_iterator(ts) + assert ts.current.type is TOKEN_BLOCK_END + assert bool(ts) + assert not bool(ts.eos) + six.advance_iterator(ts) + assert ts.current.type is TOKEN_EOF + assert not bool(ts) + assert bool(ts.eos) + + def test_iter(self): + token_types = [t.type for t in TokenStream(self.test_tokens, "foo", "bar")] + assert token_types == ['block_begin', 'block_end', ] + + class LexerTestCase(JinjaTestCase): def test_raw1(self): @@ -42,7 +68,7 @@ class LexerTestCase(JinjaTestCase): env = Environment('{%', '%}', '${', '}') tmpl = env.from_string('''{% for item in seq %}${{'foo': item}|upper}{% endfor %}''') - assert tmpl.render(seq=range(3)) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}" + assert tmpl.render(seq=list(range(3))) == "{'FOO': 0}{'FOO': 1}{'FOO': 2}" def test_comments(self): env = Environment('<!--', '-->', '{', '}') @@ -52,7 +78,7 @@ class LexerTestCase(JinjaTestCase): <li>{item}</li> <!--- endfor --> </ul>''') - assert tmpl.render(seq=range(3)) == ("<ul>\n <li>0</li>\n " + assert tmpl.render(seq=list(range(3))) == ("<ul>\n <li>0</li>\n " "<li>1</li>\n <li>2</li>\n</ul>") def test_string_escapes(self): @@ -68,11 +94,11 @@ class LexerTestCase(JinjaTestCase): def test_operators(self): from jinja2.lexer import operators - for test, expect in operators.iteritems(): + for test, expect in six.iteritems(operators): if test in '([{}])': continue stream = env.lexer.tokenize('{{ %s }}' % test) - stream.next() + six.advance_iterator(stream) assert stream.current.type == expect def test_normalizing(self): @@ -82,6 +108,19 @@ class LexerTestCase(JinjaTestCase): result = tmpl.render() assert result.replace(seq, 'X') == '1X2X3X4' + def test_trailing_newline(self): + for keep in [True, False]: + env = Environment(keep_trailing_newline=keep) + for template,expected in [ + ('', {}), + ('no\nnewline', {}), + ('with\nnewline\n', {False: 'with\nnewline'}), + ('with\nseveral\n\n\n', {False: 'with\nseveral\n\n'}), + ]: + tmpl = env.from_string(template) + expect = expected.get(keep, template) + result = tmpl.render() + assert result == expect, (keep, template, result, expect) class ParserTestCase(JinjaTestCase): @@ -92,7 +131,7 @@ class ParserTestCase(JinjaTestCase): <? for item in seq -?> <?= item ?> <?- endfor ?>''') - assert tmpl.render(seq=range(5)) == '01234' + assert tmpl.render(seq=list(range(5))) == '01234' def test_erb_syntax(self): env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>') @@ -101,7 +140,7 @@ class ParserTestCase(JinjaTestCase): <% for item in seq -%> <%= item %> <%- endfor %>''') - assert tmpl.render(seq=range(5)) == '01234' + assert tmpl.render(seq=list(range(5))) == '01234' def test_comment_syntax(self): env = Environment('<!--', '-->', '${', '}', '<!--#', '-->') @@ -110,7 +149,7 @@ class ParserTestCase(JinjaTestCase): <!-- for item in seq ---> ${item} <!--- endfor -->''') - assert tmpl.render(seq=range(5)) == '01234' + assert tmpl.render(seq=list(range(5))) == '01234' def test_balancing(self): tmpl = env.from_string('''{{{'foo':'bar'}.foo}}''') @@ -130,8 +169,8 @@ and bar comment #} % for item in seq: ${item} % endfor''') - assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \ - range(5) + assert [int(x.strip()) for x in tmpl.render(seq=list(range(5))).split()] == \ + list(range(5)) env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##') tmpl = env.from_string('''\ @@ -139,8 +178,8 @@ and bar comment #} % for item in seq: ${item} ## the rest of the stuff % endfor''') - assert [int(x.strip()) for x in tmpl.render(seq=range(5)).split()] == \ - range(5) + assert [int(x.strip()) for x in tmpl.render(seq=list(range(5))).split()] == \ + list(range(5)) def test_line_syntax_priority(self): # XXX: why is the whitespace there in front of the newline? @@ -166,7 +205,7 @@ and bar comment #} def assert_error(code, expected): try: Template(code) - except TemplateSyntaxError, e: + except TemplateSyntaxError as e: assert str(e) == expected, 'unexpected error message' else: assert False, 'that was supposed to be an error' @@ -335,9 +374,9 @@ class SyntaxTestCase(JinjaTestCase): assert tmpl.render() == 'foobarbaz' def test_notin(self): - bar = xrange(100) + bar = range(100) tmpl = env.from_string('''{{ not 42 in bar }}''') - assert tmpl.render(bar=bar) == unicode(not 42 in bar) + assert tmpl.render(bar=bar) == six.text_type(not 42 in bar) def test_implicit_subscribed_tuple(self): class Foo(object): @@ -379,9 +418,176 @@ class SyntaxTestCase(JinjaTestCase): assert tmpl.render(foo={'bar': 42}) == '42' +class LstripBlocksTestCase(JinjaTestCase): + + def test_lstrip(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% if True %}\n {% endif %}''') + assert tmpl.render() == "\n" + + def test_lstrip_trim(self): + env = Environment(lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(''' {% if True %}\n {% endif %}''') + assert tmpl.render() == "" + + def test_no_lstrip(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {%+ if True %}\n {%+ endif %}''') + assert tmpl.render() == " \n " + + def test_lstrip_endline(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' hello{% if True %}\n goodbye{% endif %}''') + assert tmpl.render() == " hello\n goodbye" + + def test_lstrip_inline(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% if True %}hello {% endif %}''') + assert tmpl.render() == 'hello ' + + def test_lstrip_nested(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% if True %}a {% if True %}b {% endif %}c {% endif %}''') + assert tmpl.render() == 'a b c ' + + def test_lstrip_left_chars(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' abc {% if True %} + hello{% endif %}''') + assert tmpl.render() == ' abc \n hello' + + def test_lstrip_embeded_strings(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {% set x = " {% str %} " %}{{ x }}''') + assert tmpl.render() == ' {% str %} ' + + def test_lstrip_preserve_leading_newlines(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string('''\n\n\n{% set hello = 1 %}''') + assert tmpl.render() == '\n\n\n' + + def test_lstrip_comment(self): + env = Environment(lstrip_blocks=True, trim_blocks=False) + tmpl = env.from_string(''' {# if True #} +hello + {#endif#}''') + assert tmpl.render() == '\nhello\n' + + def test_lstrip_angle_bracket_simple(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(''' <% if True %>hello <% endif %>''') + assert tmpl.render() == 'hello ' + + def test_lstrip_angle_bracket_comment(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string(''' <%# if True %>hello <%# endif %>''') + assert tmpl.render() == 'hello ' + + def test_lstrip_angle_bracket(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <%# regular comment %> + <% for item in seq %> +${item} ## the rest of the stuff + <% endfor %>''') + assert tmpl.render(seq=range(5)) == \ + ''.join('%s\n' % x for x in range(5)) + + def test_lstrip_angle_bracket_compact(self): + env = Environment('<%', '%>', '${', '}', '<%#', '%>', '%', '##', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <%#regular comment%> + <%for item in seq%> +${item} ## the rest of the stuff + <%endfor%>''') + assert tmpl.render(seq=range(5)) == \ + ''.join('%s\n' % x for x in range(5)) + + def test_php_syntax_with_manual(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq -?> + <?= item ?> + <?- endfor ?>''') + assert tmpl.render(seq=range(5)) == '01234' + + def test_php_syntax(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <!-- I'm a comment, I'm not interesting --> + <? for item in seq ?> + <?= item ?> + <? endfor ?>''') + assert tmpl.render(seq=range(5)) == ''.join(' %s\n' % x for x in range(5)) + + def test_php_syntax_compact(self): + env = Environment('<?', '?>', '<?=', '?>', '<!--', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ + <!-- I'm a comment, I'm not interesting --> + <?for item in seq?> + <?=item?> + <?endfor?>''') + assert tmpl.render(seq=range(5)) == ''.join(' %s\n' % x for x in range(5)) + + def test_erb_syntax(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>', + lstrip_blocks=True, trim_blocks=True) + #env.from_string('') + #for n,r in env.lexer.rules.iteritems(): + # print n + #print env.lexer.rules['root'][0][0].pattern + #print "'%s'" % tmpl.render(seq=range(5)) + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq %> + <%= item %> + <% endfor %> +''') + assert tmpl.render(seq=range(5)) == ''.join(' %s\n' % x for x in range(5)) + + def test_erb_syntax_with_manual(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %> + <% for item in seq -%> + <%= item %> + <%- endfor %>''') + assert tmpl.render(seq=range(5)) == '01234' + + def test_erb_syntax_no_lstrip(self): + env = Environment('<%', '%>', '<%=', '%>', '<%#', '%>', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ +<%# I'm a comment, I'm not interesting %> + <%+ for item in seq -%> + <%= item %> + <%- endfor %>''') + assert tmpl.render(seq=range(5)) == ' 01234' + + def test_comment_syntax(self): + env = Environment('<!--', '-->', '${', '}', '<!--#', '-->', + lstrip_blocks=True, trim_blocks=True) + tmpl = env.from_string('''\ +<!--# I'm a comment, I'm not interesting -->\ +<!-- for item in seq ---> + ${item} +<!--- endfor -->''') + assert tmpl.render(seq=range(5)) == '01234' + def suite(): suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TokenStreamTestCase)) suite.addTest(unittest.makeSuite(LexerTestCase)) suite.addTest(unittest.makeSuite(ParserTestCase)) suite.addTest(unittest.makeSuite(SyntaxTestCase)) + suite.addTest(unittest.makeSuite(LstripBlocksTestCase)) return suite diff --git a/jinja2/testsuite/loader.py b/jinja2/testsuite/loader.py index f62ec92..9368698 100644 --- a/jinja2/testsuite/loader.py +++ b/jinja2/testsuite/loader.py @@ -93,6 +93,13 @@ class LoaderTestCase(JinjaTestCase): assert 'two' not in env.cache assert 'three' in env.cache + def test_dict_loader_cache_invalidates(self): + mapping = {'foo': "one"} + env = Environment(loader=loaders.DictLoader(mapping)) + assert env.get_template('foo').render() == "one" + mapping['foo'] = "two" + assert env.get_template('foo').render() == "two" + def test_split_template_path(self): assert split_template_path('foo/bar') == ['foo', 'bar'] assert split_template_path('./foo/bar') == ['foo', 'bar'] diff --git a/jinja2/testsuite/regression.py b/jinja2/testsuite/regression.py index 4db9076..a4e6631 100644 --- a/jinja2/testsuite/regression.py +++ b/jinja2/testsuite/regression.py @@ -14,6 +14,7 @@ from jinja2.testsuite import JinjaTestCase from jinja2 import Template, Environment, DictLoader, TemplateSyntaxError, \ TemplateNotFound, PrefixLoader +import six env = Environment() @@ -118,7 +119,7 @@ class BugTestCase(JinjaTestCase): ''') - assert tmpl.render().split() == map(unicode, range(1, 11)) * 5 + assert tmpl.render().split() == [six.text_type(x) for x in range(1, 11)] * 5 def test_weird_inline_comment(self): env = Environment(line_statement_prefix='%') @@ -236,17 +237,40 @@ class BugTestCase(JinjaTestCase): {% endfor %} """) + def test_else_loop_bug(self): + t = Template(''' + {% for x in y %} + {{ loop.index0 }} + {% else %} + {% for i in range(3) %}{{ i }}{% endfor %} + {% endfor %} + ''') + self.assertEqual(t.render(y=[]).strip(), '012') + def test_correct_prefix_loader_name(self): env = Environment(loader=PrefixLoader({ 'foo': DictLoader({}) })) try: env.get_template('foo/bar.html') - except TemplateNotFound, e: + except TemplateNotFound as e: assert e.name == 'foo/bar.html' else: assert False, 'expected error here' + def test_contextfunction_callable_classes(self): + from jinja2.utils import contextfunction + class CallableClass(object): + @contextfunction + def __call__(self, ctx): + return ctx.resolve('hello') + + tpl = Template("""{{ callableclass() }}""") + output = tpl.render(callableclass = CallableClass(), hello = 'TEST') + expected = 'TEST' + + self.assert_equal(output, expected) + def suite(): suite = unittest.TestSuite() diff --git a/jinja2/testsuite/security.py b/jinja2/testsuite/security.py index 4518eac..c892fed 100644 --- a/jinja2/testsuite/security.py +++ b/jinja2/testsuite/security.py @@ -18,6 +18,7 @@ from jinja2.sandbox import SandboxedEnvironment, \ from jinja2 import Markup, escape from jinja2.exceptions import SecurityError, TemplateSyntaxError, \ TemplateRuntimeError +import six class PrivateStuff(object): @@ -76,7 +77,7 @@ class SandboxTestCase(JinjaTestCase): # adding two strings should escape the unsafe one unsafe = '<script type="application/x-some-script">alert("foo");</script>' safe = Markup('<em>username</em>') - assert unsafe + safe == unicode(escape(unsafe)) + unicode(safe) + assert unsafe + safe == six.text_type(escape(unsafe)) + six.text_type(safe) # string interpolations are safe to use too assert Markup('<em>%s</em>') % '<bad user>' == \ @@ -114,7 +115,7 @@ class SandboxTestCase(JinjaTestCase): '{{ say_hello("<blink>foo</blink>") }}') escaped_out = '<p>Hello <blink>foo</blink>!</p>' assert t.render() == escaped_out - assert unicode(t.module) == escaped_out + assert six.text_type(t.module) == escaped_out assert escape(t.module) == escaped_out assert t.module.say_hello('<blink>foo</blink>') == escaped_out assert escape(t.module.say_hello('<blink>foo</blink>')) == escaped_out @@ -136,7 +137,7 @@ class SandboxTestCase(JinjaTestCase): t = env.from_string('{{ %s }}' % expr) try: t.render(ctx) - except TemplateRuntimeError, e: + except TemplateRuntimeError as e: pass else: self.fail('expected runtime error') @@ -153,7 +154,7 @@ class SandboxTestCase(JinjaTestCase): t = env.from_string('{{ %s }}' % expr) try: t.render(ctx) - except TemplateRuntimeError, e: + except TemplateRuntimeError as e: pass else: self.fail('expected runtime error') diff --git a/jinja2/testsuite/utils.py b/jinja2/testsuite/utils.py index be2e902..cab9b09 100644 --- a/jinja2/testsuite/utils.py +++ b/jinja2/testsuite/utils.py @@ -60,8 +60,8 @@ class MarkupLeakTestCase(JinjaTestCase): def test_markup_leaks(self): counts = set() - for count in xrange(20): - for item in xrange(1000): + for count in range(20): + for item in range(1000): escape("foo") escape("<foo>") escape(u"foo") diff --git a/jinja2/utils.py b/jinja2/utils.py index 1e0bb81..3440979 100644 --- a/jinja2/utils.py +++ b/jinja2/utils.py @@ -11,6 +11,8 @@ import re import sys import errno +import six +from six.moves import map try: from urllib.parse import quote_from_bytes as url_quote except ImportError: @@ -18,16 +20,17 @@ except ImportError: try: from thread import allocate_lock except ImportError: - from dummy_thread import allocate_lock + try: + from _thread import allocate_lock # py 3 + except ImportError: + from dummy_thread import allocate_lock from collections import deque -from itertools import imap - _word_split_re = re.compile(r'(\s+)') _punctuation_re = re.compile( '^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % ( - '|'.join(imap(re.escape, ('(', '<', '<'))), - '|'.join(imap(re.escape, ('.', ',', ')', '>', '\n', '>'))) + '|'.join(map(re.escape, ('(', '<', '<'))), + '|'.join(map(re.escape, ('.', ',', ')', '>', '\n', '>'))) ) ) _simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$') @@ -42,38 +45,7 @@ missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})() # internal code internal_code = set() - -# concatenate a list of strings and convert them to unicode. -# unfortunately there is a bug in python 2.4 and lower that causes -# unicode.join trash the traceback. -_concat = u''.join -try: - def _test_gen_bug(): - raise TypeError(_test_gen_bug) - yield None - _concat(_test_gen_bug()) -except TypeError, _error: - if not _error.args or _error.args[0] is not _test_gen_bug: - def concat(gen): - try: - return _concat(list(gen)) - except Exception: - # this hack is needed so that the current frame - # does not show up in the traceback. - exc_type, exc_value, tb = sys.exc_info() - raise exc_type, exc_value, tb.tb_next - else: - concat = _concat - del _test_gen_bug, _error - - -# for python 2.x we create outselves a next() function that does the -# basics without exception catching. -try: - next = next -except NameError: - def next(x): - return x.next() +concat = u''.join # if this python version is unable to deal with unicode filenames @@ -81,7 +53,7 @@ except NameError: # This is used in a couple of places. As far as Jinja is concerned # filenames are unicode *or* bytestrings in 2.x and unicode only in # 3.x because compile cannot handle bytes -if sys.version_info < (3, 0): +if sys.version_info[0] < 3: def _encode_filename(filename): if isinstance(filename, unicode): return filename.encode('utf-8') @@ -104,8 +76,8 @@ def _func(): yield None FunctionType = type(_func) GeneratorType = type(_func()) -MethodType = type(_C.method) -CodeType = type(_C.method.func_code) +MethodType = type(_C().method) +CodeType = type(_C.method.__code__) try: raise TypeError() except TypeError: @@ -132,7 +104,7 @@ def contextfunction(f): def evalcontextfunction(f): - """This decoraotr can be used to mark a function or method as an eval + """This decorator can be used to mark a function or method as an eval context callable. This is similar to the :func:`contextfunction` but instead of passing the context, an evaluation context object is passed. For more information about the eval context, see @@ -156,7 +128,7 @@ def environmentfunction(f): def internalcode(f): """Marks the function as internally used""" - internal_code.add(f.func_code) + internal_code.add(f.__code__) return f @@ -195,7 +167,7 @@ def clear_caches(): def import_string(import_name, silent=False): - """Imports an object based on a string. This use useful if you want to + """Imports an object based on a string. This is useful if you want to use import paths as endpoints or something similar. An import path can be specified either in dotted notation (``xml.sax.saxutils.escape``) or with a colon as object delimiter (``xml.sax.saxutils:escape``). @@ -226,7 +198,7 @@ def open_if_exists(filename, mode='rb'): """ try: return open(filename, mode) - except IOError, e: + except IOError as e: if e.errno not in (errno.ENOENT, errno.EISDIR): raise @@ -275,7 +247,7 @@ def urlize(text, trim_url_limit=None, nofollow=False): trim_url = lambda x, limit=trim_url_limit: limit is not None \ and (x[:limit] + (len(x) >=limit and '...' or '')) or x - words = _word_split_re.split(unicode(escape(text))) + words = _word_split_re.split(six.text_type(escape(text))) nofollow_attr = nofollow and ' rel="nofollow"' or '' for i, word in enumerate(words): match = _punctuation_re.match(word) @@ -284,6 +256,7 @@ def urlize(text, trim_url_limit=None, nofollow=False): if middle.startswith('www.') or ( '@' not in middle and not middle.startswith('http://') and + not middle.startswith('https://') and len(middle) > 0 and middle[0] in _letters + _digits and ( middle.endswith('.org') or @@ -311,7 +284,7 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): words = LOREM_IPSUM_WORDS.split() result = [] - for _ in xrange(n): + for _ in range(n): next_capitalized = True last_comma = last_fullstop = 0 word = None @@ -319,7 +292,7 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): p = [] # each paragraph contains out of 20 to 100 words. - for idx, _ in enumerate(xrange(randrange(min, max))): + for idx, _ in enumerate(range(randrange(min, max))): while True: word = choice(words) if word != last: @@ -361,11 +334,11 @@ def unicode_urlencode(obj, charset='utf-8'): If non strings are provided they are converted to their unicode representation first. """ - if not isinstance(obj, basestring): - obj = unicode(obj) - if isinstance(obj, unicode): + if not isinstance(obj, six.string_types): + obj = six.text_type(obj) + if isinstance(obj, six.text_type): obj = obj.encode(charset) - return unicode(url_quote(obj)) + return six.text_type(url_quote(obj)) class LRUCache(object): @@ -385,18 +358,10 @@ class LRUCache(object): # alias all queue methods for faster lookup self._popleft = self._queue.popleft self._pop = self._queue.pop - if hasattr(self._queue, 'remove'): - self._remove = self._queue.remove + self._remove = self._queue.remove self._wlock = allocate_lock() self._append = self._queue.append - def _remove(self, obj): - """Python 2.4 compatibility.""" - for idx, item in enumerate(self._queue): - if item == obj: - del self._queue[idx] - break - def __getstate__(self): return { 'capacity': self.capacity, @@ -412,7 +377,7 @@ class LRUCache(object): return (self.capacity,) def copy(self): - """Return an shallow copy of the instance.""" + """Return a shallow copy of the instance.""" rv = self.__class__(self.capacity) rv._mapping.update(self._mapping) rv._queue = deque(self._queue) @@ -429,11 +394,15 @@ class LRUCache(object): """Set `default` if the key is not in the cache otherwise leave unchanged. Return the value of this key. """ + self._wlock.acquire() try: - return self[key] - except KeyError: - self[key] = default - return default + try: + return self[key] + except KeyError: + self[key] = default + return default + finally: + self._wlock.release() def clear(self): """Clear the cache.""" @@ -462,19 +431,23 @@ class LRUCache(object): """Get an item from the cache. Moves the item up so that it has the highest priority then. - Raise an `KeyError` if it does not exist. + Raise a `KeyError` if it does not exist. """ - rv = self._mapping[key] - if self._queue[-1] != key: - try: - self._remove(key) - except ValueError: - # if something removed the key from the container - # when we read, ignore the ValueError that we would - # get otherwise. - pass - self._append(key) - return rv + self._wlock.acquire() + try: + rv = self._mapping[key] + if self._queue[-1] != key: + try: + self._remove(key) + except ValueError: + # if something removed the key from the container + # when we read, ignore the ValueError that we would + # get otherwise. + pass + self._append(key) + return rv + finally: + self._wlock.release() def __setitem__(self, key, value): """Sets the value for an item. Moves the item up so that it @@ -483,11 +456,7 @@ class LRUCache(object): self._wlock.acquire() try: if key in self._mapping: - try: - self._remove(key) - except ValueError: - # __getitem__ is not locked, it might happen - pass + self._remove(key) elif len(self._mapping) == self.capacity: del self._mapping[self._popleft()] self._append(key) @@ -497,7 +466,7 @@ class LRUCache(object): def __delitem__(self, key): """Remove an item from the cache dict. - Raise an `KeyError` if it does not exist. + Raise a `KeyError` if it does not exist. """ self._wlock.acquire() try: @@ -557,7 +526,7 @@ except ImportError: pass -class Cycler(object): +class Cycler(six.Iterator): """A cycle helper for templates.""" def __init__(self, *items): @@ -575,7 +544,7 @@ class Cycler(object): """Returns the current item.""" return self.items[self.pos] - def next(self): + def __next__(self): """Goes one item ahead and returns it.""" rv = self.current self.pos = (self.pos + 1) % len(self.items) @@ -598,7 +567,7 @@ class Joiner(object): # try markupsafe first, if that fails go with Jinja2's bundled version # of markupsafe. Markupsafe was previously Jinja2's implementation of -# the Markup object but was moved into a separate package in a patchleve +# the Markup object but was moved into a separate package in a patchlevel # release try: from markupsafe import Markup, escape, soft_unicode @@ -606,15 +575,4 @@ except ImportError: from jinja2._markupsafe import Markup, escape, soft_unicode -# partials -try: - from functools import partial -except ImportError: - class partial(object): - def __init__(self, _func, *args, **kwargs): - self._func = _func - self._args = args - self._kwargs = kwargs - def __call__(self, *args, **kwargs): - kwargs.update(self._kwargs) - return self._func(*(self._args + args), **kwargs) +from functools import partial @@ -37,24 +37,8 @@ For more informations visit the new `Jinja2 webpage`_ and `documentation`_. """ import sys -from setuptools import setup, Extension, Feature +from setuptools import setup -debugsupport = Feature( - 'optional C debug support', - standard=False, - ext_modules = [ - Extension('jinja2._debugsupport', ['jinja2/_debugsupport.c']), - ], -) - - -# tell distribute to use 2to3 with our own fixers. -extra = {} -if sys.version_info >= (3, 0): - extra.update( - use_2to3=True, - use_2to3_fixers=['custom_fixers'] - ) # ignore the old '--with-speedups' flag try: @@ -62,11 +46,10 @@ try: except ValueError: pass else: - sys.argv[speedups_pos] = '--with-debugsupport' + sys.argv[speedups_pos] = '' sys.stderr.write('*' * 74 + '\n') sys.stderr.write('WARNING:\n') - sys.stderr.write(' the --with-speedups flag is deprecated, assuming ' - '--with-debugsupport\n') + sys.stderr.write(' the --with-speedups flag is deprecated\n') sys.stderr.write(' For the actual speedups install the MarkupSafe ' 'package.\n') sys.stderr.write('*' * 74 + '\n') @@ -98,13 +81,12 @@ setup( ], packages=['jinja2', 'jinja2.testsuite', 'jinja2.testsuite.res', 'jinja2._markupsafe'], + install_requires=['six>=1.3.0'], extras_require={'i18n': ['Babel>=0.8']}, test_suite='jinja2.testsuite.suite', include_package_data=True, entry_points=""" [babel.extractors] jinja2 = jinja2.ext:babel_extract[i18n] - """, - features={'debugsupport': debugsupport}, - **extra + """ ) @@ -0,0 +1,5 @@ +[tox] +envlist = py26, py27, pypy, py33 + +[testenv] +commands = python setup.py test |