even more tests, fixed severe bug with autoescaping.

--HG--
branch : trunk

15 files changed, 209 insertions, 28 deletions

diff --git a/docs/api.rst b/docs/api.rst
index 95064b5..cba97ea 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -56,8 +56,8 @@ sequence which is per default UNIX style (``\n``).
 High Level API
 --------------
 
-.. autoclass:: jinja2.environment.Environment([options])
-    :members: from_string, get_template, join_path, parse, lex, extend
+.. autoclass:: Environment([options])
+    :members: from_string, get_template, join_path, extend
 
     .. attribute:: shared
 
@@ -96,8 +96,42 @@ High Level API
 
     .. automethod:: overlay([options])
 
+    .. method:: undefined([hint,] [obj,] name[, exc])
 
-.. autoclass:: jinja2.Template
+        Creates a new :class:`Undefined` object for `name`.  This is useful
+        for filters or functions that may return undefined objects for
+        some operations.  All parameters except of `hint` should be provided
+        as keyword parameters for better readability.  The `hint` is used as
+        error message for the exception if provided, otherwise the error
+        message generated from `obj` and `name` automatically.  The exception
+        provided as `exc` is raised if something with the generated undefined
+        object is done that the undefined object does not allow.  The default
+        exception is :exc:`UndefinedError`.  If a `hint` is provided the
+        `name` may be ommited.
+
+        The most common way to create an undefined object is by providing
+        a name only::
+
+            return environment.undefined(name='some_name')
+
+        This means that the name `some_name` is not defined.  If the name
+        was from an attribute of an object it makes sense to tell the
+        undefined object the holder object to improve the error message::
+
+            if not hasattr(obj, 'attr'):
+                return environment.undefined(obj=obj, name='attr')
+
+        For a more complex example you can provide a hint.  For example
+        the :func:`first` filter creates an undefined object that way::
+
+            return environment.undefined('no first item, sequence was empty')            
+
+        If it the `name` or `obj` is known (for example because an attribute
+        was accessed) it shold be passed to the undefined object, even if
+        a custom `hint` is provided.  This gives undefined objects the
+        possibility to enhance the error message.
+
+.. autoclass:: Template
     :members: make_module, module, new_context
 
     .. attribute:: globals
@@ -111,6 +145,11 @@ High Level API
         The loading name of the template.  If the template was loaded from a
         string this is `None`.
 
+    .. attribute:: filename
+
+        The filename of the template on the file system if it was loaded from
+        there.  Otherwise this is `None`.
+
     .. automethod:: render([context])
 
     .. automethod:: generate([context])
@@ -125,7 +164,7 @@ High Level API
 .. _identifier-naming:
 
 Notes on Identifiers
-~~~~~~~~~~~~~~~~~~~~
+--------------------
 
 Jinja2 uses the regular Python 2.x naming rules.  Valid identifiers have to
 match ``[a-zA-Z_][a-zA-Z0-9_]*``.  As a matter of fact non ASCII characters
@@ -153,11 +192,13 @@ others fail.
 The closest to regular Python behavior is the `StrictUndefined` which
 disallows all operations beside testing if it's an undefined object.
 
-.. autoclass:: jinja2.runtime.Undefined
+.. autoclass:: jinja2.runtime.Undefined()
+
+.. autoclass:: jinja2.runtime.DebugUndefined()
 
-.. autoclass:: jinja2.runtime.DebugUndefined
+.. autoclass:: jinja2.runtime.StrictUndefined()
 
-.. autoclass:: jinja2.runtime.StrictUndefined
+Undefined objects are created by calling :attr:`undefined`.
 
 
 The Context
@@ -170,8 +211,9 @@ The Context
 
         A dict of read only, global variables the template looks up.  These
         can either come from another :class:`Context`, from the
-        :attr:`Environment.globals` or :attr:`Template.globals`.  It must not
-        be altered.
+        :attr:`Environment.globals` or :attr:`Template.globals` or points
+        to a dict created by combining the globals with the variables
+        passed to the render function.  It must not be altered.
 
     .. attribute:: vars
 
@@ -399,3 +441,40 @@ context.  This is the place where you can put variables and functions
 that should be available all the time.  Additionally :attr:`Template.globals`
 exist that are variables available to a specific template that are available
 to all :meth:`~Template.render` calls.
+
+
+Low Level API
+-------------
+
+The low level API exposes functionality that can be useful to understand some
+implementation details, debugging purposes or advanced :ref:`extension
+<jinja-extensions>` techniques.
+
+.. automethod:: Environment.lex
+
+.. automethod:: Environment.parse
+
+.. automethod:: Template.new_context
+
+.. method:: Template.root_render_func(context)
+
+    This is the low level render function.  It's passed a :class:`Context`
+    that has to be created by :meth:`new_context` of the same template or
+    a compatible template.  This render function is generated by the
+    compiler from the template code and returns a generator that yields
+    unicode strings.
+
+    If an exception in the template code happens the template engine will
+    not rewrite the exception but pass through the original one.  As a
+    matter of fact this function should only be called from within a
+    :meth:`render` / :meth:`generate` / :meth:`stream` call.
+
+.. attribute:: Template.blocks
+
+    A dict of block render functions.  Each of these functions works exactly
+    like the :meth:`root_render_func` with the same limitations.
+
+.. attribute:: Template.is_up_to_date
+
+    This attribute is `False` if there is a newer version of the template
+    available, otherwise `True`.
diff --git a/docs/faq.rst b/docs/faq.rst
index 4b4fab4..dc75f98 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -5,7 +5,6 @@ This page answers some of the often asked questions about Jinja.
 
 .. highlight:: html+jinja
 
-
 Why is it called Jinja?
 -----------------------
 
@@ -21,7 +20,11 @@ performance of a template depends on many factors and you would have to
 benchmark different engines in different situations.  The benchmarks from the
 testsuite show that Jinja2 has a similar performance to `Mako`_ and is more
 than 20 times faster than Django's template engine or Genshi.  These numbers
-should be taken with tons of salt!
+should be taken with tons of salt as the benchmarks that took these numbers
+only test a few performance related situations such as looping.  They are
+not a good indicator for the templates used in the average application.
+Additionally you should keep in mind that for most web applications
+templates are clearly not the bottleneck.
 
 .. _Mako: http://www.makotemplates.org/
 
@@ -121,3 +124,24 @@ If you want to modify the context write a function that returns a variable
 instead that one can assign to a variable by using set::
 
     {% set comments = get_latest_comments() %}
+
+I don't have the _speedups Module.  Is Jinja slower now?
+--------------------------------------------------------
+
+To achieve a good performance with automatic escaping enabled the escaping
+function is implemented also written in pure C and used if Jinja2 was
+installed with the speedups module which automatically happens if a C
+compiled is available on the system.  It won't affect templates without
+auto escaping much if that feature is not enabled.  You may however
+experience werid tracebacks if you are using a Python installation, for
+more information see the next FAQ item.
+
+My tracebacks look weird.  What's happening?
+--------------------------------------------
+
+If the speedups module is not compiled and you are using a Python installation
+without ctypes (Python 2.4 without ctypes, Jython or Google's AppEngine)
+Jinja2 is unable to provide correct debugging information and the traceback
+may be incomplete.  There is currently no good workaround for Jython or
+the AppEngine as ctypes is unavailable there and it's not possible to use
+the speedups extension.
diff --git a/docs/jinjaext.py b/docs/jinjaext.py
index 1ed6d35..8a15d65 100644
--- a/docs/jinjaext.py
+++ b/docs/jinjaext.py
@@ -48,7 +48,7 @@ class JinjaStyle(Style):
         Keyword:                    'bold #B80000',
         Keyword.Type:               '#808080',
 
-        Operator.Word:              '#333333',
+        Operator.Word:              'bold #B80000',
 
         Name.Builtin:               '#333333',
         Name.Function:              '#333333',
diff --git a/examples/rwbench/rwbench.py b/examples/rwbench/rwbench.py
index 2483a8d..1f3e387 100644
--- a/examples/rwbench/rwbench.py
+++ b/examples/rwbench/rwbench.py
@@ -81,4 +81,4 @@ if __name__ == '__main__':
                   stmt='bench()')
         sys.stdout.write(' >> %-20s<running>' % test)
         sys.stdout.flush()
-        sys.stdout.write('\r    %-20s%.4f seconds\n' % (test, t.timeit(number=50) / 50))
+        sys.stdout.write('\r    %-20s%.4f seconds\n' % (test, t.timeit(number=200) / 200))
diff --git a/jinja2/_speedups.c b/jinja2/_speedups.c
index 61bdec7..112e600 100644
--- a/jinja2/_speedups.c
+++ b/jinja2/_speedups.c
@@ -2,10 +2,11 @@
  * jinja2._speedups
  * ~~~~~~~~~~~~~~~~
  *
- * This module implements a few functions in C for better performance.  It
- * also defines a `tb_set_next` function that is used to patch the debug
- * traceback.  If the speedups module is not compiled a ctypes implementation
- * is used.
+ * This module implements functions for automatic escaping in C for better
+ * performance.  Additionally it defines a `tb_set_next` function to patch the
+ * debug traceback.  If the speedups module is not compiled a ctypes
+ * implementation of `tb_set_next` and Python implementations of the other
+ * functions are used.
  *
  * :copyright: 2008 by Armin Ronacher, Mickaël Guérin.
  * :license: BSD.
diff --git a/jinja2/compiler.py b/jinja2/compiler.py
index 83afc34..9d68e4c 100644
--- a/jinja2/compiler.py
+++ b/jinja2/compiler.py
@@ -680,7 +680,7 @@ class CodeGenerator(NodeVisitor):
                 self.writeline('if parent_template is not None:')
             self.indent()
             self.writeline('for event in parent_template.'
-                           '_root_render_func(context):')
+                           'root_render_func(context):')
             self.indent()
             self.writeline('yield event')
             self.outdent(2 + (not self.has_known_extends))
@@ -784,7 +784,7 @@ class CodeGenerator(NodeVisitor):
             self.writeline('template = environment.get_template(', node)
             self.visit(node.template, frame)
             self.write(', %r)' % self.name)
-            self.writeline('for event in template._root_render_func('
+            self.writeline('for event in template.root_render_func('
                            'template.new_context(context.parent, True)):')
         else:
             self.writeline('for event in environment.get_template(', node)
@@ -1191,6 +1191,9 @@ class CodeGenerator(NodeVisitor):
         else:
             self.write(repr(val))
 
+    def visit_TemplateData(self, node, frame):
+        self.write(repr(node.as_const()))
+
     def visit_Tuple(self, node, frame):
         self.write('(')
         idx = -1
diff --git a/jinja2/environment.py b/jinja2/environment.py
index 45d684d..5ec8cb5 100644
--- a/jinja2/environment.py
+++ b/jinja2/environment.py
@@ -405,7 +405,7 @@ class Environment(object):
 
     def make_globals(self, d):
         """Return a dict for the globals."""
-        if d is None:
+        if not d:
             return self.globals
         return dict(self.globals, **d)
 
@@ -482,7 +482,7 @@ class Template(object):
         t.blocks = namespace['blocks']
 
         # render function and module 
-        t._root_render_func = namespace['root']
+        t.root_render_func = namespace['root']
         t._module = None
 
         # debug and loader helpers
@@ -503,7 +503,7 @@ class Template(object):
         """
         vars = dict(*args, **kwargs)
         try:
-            return concat(self._root_render_func(self.new_context(vars)))
+            return concat(self.root_render_func(self.new_context(vars)))
         except:
             from jinja2.debug import translate_exception
             exc_type, exc_value, tb = translate_exception(sys.exc_info())
@@ -525,7 +525,7 @@ class Template(object):
         """
         vars = dict(*args, **kwargs)
         try:
-            for event in self._root_render_func(self.new_context(vars)):
+            for event in self.root_render_func(self.new_context(vars)):
                 yield event
         except:
             from jinja2.debug import translate_exception
@@ -533,7 +533,7 @@ class Template(object):
             raise exc_type, exc_value, tb
 
     def new_context(self, vars=None, shared=False):
-        """Create a new template context for this template.  The vars
+        """Create a new :class:`Context` for this template.  The vars
         provided will be passed to the template.  Per default the globals
         are added to the context, if shared is set to `True` the data
         provided is used as parent namespace.  This is used to share the
@@ -611,12 +611,12 @@ class TemplateModule(object):
     """
 
     def __init__(self, template, context):
-        self._body_stream = list(template._root_render_func(context))
+        self._body_stream = list(template.root_render_func(context))
         self.__dict__.update(context.get_exported())
         self.__name__ = template.name
 
-    __html__ = lambda x: Markup(concat(x._body_stream))
     __unicode__ = lambda x: concat(x._body_stream)
+    __html__ = lambda x: Markup(concat(x._body_stream))
 
     def __str__(self):
         return unicode(self).encode('utf-8')
diff --git a/jinja2/filters.py b/jinja2/filters.py
index ed3d57c..de15b53 100644
--- a/jinja2/filters.py
+++ b/jinja2/filters.py
@@ -598,6 +598,11 @@ def do_mark_safe(value):
     return Markup(value)
 
 
+def do_mark_unsafe(value):
+    """Mark a value as unsafe.  This is the reverse operation for :func:`safe`."""
+    return unicode(value)
+
+
 def do_reverse(value):
     """Reverse the object or return an iterator the iterates over it the other
     way round.
diff --git a/jinja2/nodes.py b/jinja2/nodes.py
index 9eb5460..0cccddf 100644
--- a/jinja2/nodes.py
+++ b/jinja2/nodes.py
@@ -431,6 +431,16 @@ class Const(Literal):
         return cls(value, lineno=lineno, environment=environment)
 
 
+class TemplateData(Literal):
+    """A constant template string."""
+    fields = ('data',)
+
+    def as_const(self):
+        if self.environment.autoescape:
+            return Markup(self.data)
+        return self.data
+
+
 class Tuple(Literal):
     """For loop unpacking and some other things like multiple arguments
     for subscripts.  Like for :class:`Name` `ctx` specifies if the tuple
diff --git a/jinja2/parser.py b/jinja2/parser.py
index 8ca1bd2..fcc684b 100644
--- a/jinja2/parser.py
+++ b/jinja2/parser.py
@@ -723,7 +723,8 @@ class Parser(object):
             token = self.stream.current
             if token.type is 'data':
                 if token.value:
-                    add_data(nodes.Const(token.value, lineno=token.lineno))
+                    add_data(nodes.TemplateData(token.value,
+                                                lineno=token.lineno))
                 self.stream.next()
             elif token.type is 'variable_begin':
                 self.stream.next()
diff --git a/jinja2/runtime.py b/jinja2/runtime.py
index 1325b17..590bed9 100644
--- a/jinja2/runtime.py
+++ b/jinja2/runtime.py
@@ -110,7 +110,7 @@ class Context(object):
 
     def get_all(self):
         """Return a copy of the complete context as dict including the
-        global variables.
+        exported variables.
         """
         return dict(self.parent, **self.vars)
 
diff --git a/tests/loaderres/templates/broken.html b/tests/loaderres/templates/broken.html
new file mode 100644
index 0000000..77669fa
--- /dev/null
+++ b/tests/loaderres/templates/broken.html
@@ -0,0 +1,3 @@
+Before
+{{ fail() }}
+After
diff --git a/tests/loaderres/templates/syntaxerror.html b/tests/loaderres/templates/syntaxerror.html
new file mode 100644
index 0000000..f21b817
--- /dev/null
+++ b/tests/loaderres/templates/syntaxerror.html
@@ -0,0 +1,4 @@
+Foo
+{% for item in broken %}
+  ...
+{% endif %}
diff --git a/tests/test_debug.py b/tests/test_debug.py
new file mode 100644
index 0000000..2363fe2
--- /dev/null
+++ b/tests/test_debug.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+    Test debug interface
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Tests the traceback rewriter.
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: BSD.
+"""
+from jinja2 import Environment
+from test_loaders import filesystem_loader
+
+
+env = Environment(loader=filesystem_loader)
+
+
+test_runtime_error = '''
+>>> tmpl = MODULE.env.get_template('broken.html')
+>>> tmpl.render(fail=lambda: 1 / 0)
+Traceback (most recent call last):
+  File "loaderres/templates/broken.html", line 2, in top-level template code
+    {{ fail() }}
+  File "<doctest test_runtime_error[1]>", line 1, in <lambda>
+    tmpl.render(fail=lambda: 1 / 0)
+ZeroDivisionError: integer division or modulo by zero
+'''
+
+
+test_syntax_error = '''
+>>> tmpl = MODULE.env.get_template('syntaxerror.html')
+Traceback (most recent call last):
+  ...
+  File "loaderres/templates/syntaxerror.html", line 4, in <module>
+    {% endif %}
+TemplateSyntaxError: unknown tag 'endif' (syntaxerror.html, line 4)
+'''
diff --git a/tests/test_security.py b/tests/test_security.py
index 0da2df2..68b1515 100644
--- a/tests/test_security.py
+++ b/tests/test_security.py
@@ -6,6 +6,7 @@
     :copyright: 2007 by Armin Ronacher.
     :license: BSD, see LICENSE for more details.
 """
+from jinja2 import Environment
 from jinja2.sandbox import SandboxedEnvironment, \
      ImmutableSandboxedEnvironment, unsafe
 from jinja2 import Markup, escape
@@ -118,3 +119,16 @@ def test_markup_operations():
     assert escape('"<>&\'') == '&#34;&lt;&gt;&amp;&#39;'
     assert Markup("<em>Foo &amp; Bar</em>").striptags() == "Foo & Bar"
     assert Markup("&lt;test&gt;").unescape() == "<test>"
+
+
+def test_template_data():
+    env = Environment(autoescape=True)
+    t = env.from_string('{% macro say_hello(name) %}'
+                        '<p>Hello {{ name }}!</p>{% endmacro %}'
+                        '{{ say_hello("<blink>foo</blink>") }}')
+    escaped_out = '<p>Hello &lt;blink&gt;foo&lt;/blink&gt;!</p>'
+    assert t.render() == escaped_out
+    assert unicode(t.module) == escaped_out
+    assert escape(t.module) == escaped_out
+    assert t.module.say_hello('<blink>foo</blink>') == escaped_out
+    assert escape(t.module.say_hello('<blink>foo</blink>')) == escaped_out