Add support for the Environment to optionally return native types.

This works by having an alternate CodeGenerator that avoids doing to_string
after the yield statement and a new version of concat that handles the returned
generator with a bit more "intelligence".

5 files changed, 380 insertions, 0 deletions

diff --git a/CHANGES b/CHANGES
index 6276b11..4b0f534 100644
--- a/CHANGES
+++ b/CHANGES
@@ -28,11 +28,14 @@ Version 2.10
 - Add ``min`` and ``max`` filters. (`#475`_)
 - Add tests for all comparison operators: ``eq``, ``ne``, ``lt``, ``le``,
   ``gt``, ``ge``. (`#665`_)
+- Add a ``NativeEnvironment`` that renders templates to native Python types
+  instead of strings. (`#708`_)
 
 .. _#469: https://github.com/pallets/jinja/pull/469
 .. _#475: https://github.com/pallets/jinja/pull/475
 .. _#478: https://github.com/pallets/jinja/pull/478
 .. _#665: https://github.com/pallets/jinja/pull/665
+.. _#708: https://github.com/pallets/jinja/pull/708
 
 Version 2.9.6
 -------------
diff --git a/docs/contents.rst.inc b/docs/contents.rst.inc
index 7ee6870..467d4ce 100644
--- a/docs/contents.rst.inc
+++ b/docs/contents.rst.inc
@@ -7,6 +7,7 @@ Jinja2 Documentation
    intro
    api
    sandbox
+   nativetypes
    templates
    extensions
    integration
diff --git a/docs/nativetypes.rst b/docs/nativetypes.rst
new file mode 100644
index 0000000..1a08700
--- /dev/null
+++ b/docs/nativetypes.rst
@@ -0,0 +1,64 @@
+.. module:: jinja2.nativetypes
+
+.. _nativetypes:
+
+Native Python Types
+===================
+
+The default :class:`~jinja2.Environment` renders templates to strings. With
+:class:`NativeEnvironment`, rendering a template produces a native Python type.
+This is useful if you are using Jinja outside the context of creating text
+files. For example, your code may have an intermediate step where users may use
+templates to define values that will then be passed to a traditional string
+environment.
+
+Examples
+--------
+
+Adding two values results in an integer, not a string with a number:
+
+>>> env = NativeEnvironment()
+>>> t = env.from_string('{{ x + y }}')
+>>> result = t.render(x=4, y=2)
+>>> print(result)
+6
+>>> print(type(result))
+int
+
+Rendering list syntax produces a list:
+
+>>> t = env.from_string('[{% for item in data %}{{ item + 1 }},{% endfor %}]')
+>>> result = t.render(data=range(5))
+>>> print(result)
+[1, 2, 3, 4, 5]
+>>> print(type(result))
+list
+
+Rendering something that doesn't look like a Python literal produces a string:
+
+>>> t = env.from_string('{{ x }} * {{ y }}')
+>>> result = t.render(x=4, y=2)
+>>> print(result)
+4 * 2
+>>> print(type(result))
+str
+
+Rendering a Python object produces that object as long as it is the only node:
+
+>>> class Foo:
+...     def __init__(self, value):
+...         self.value = value
+...
+>>> result = env.from_string('{{ x }}').render(x=Foo(15))
+>>> print(type(result).__name__)
+Foo
+>>> print(result.value)
+15
+
+API
+---
+
+.. autoclass:: NativeEnvironment([options])
+
+.. autoclass:: NativeTemplate([options])
+    :members: render
diff --git a/jinja2/nativetypes.py b/jinja2/nativetypes.py
new file mode 100644
index 0000000..3aca683
--- /dev/null
+++ b/jinja2/nativetypes.py
@@ -0,0 +1,218 @@
+import sys
+from ast import literal_eval
+from itertools import islice, chain
+from jinja2 import nodes
+from jinja2._compat import text_type
+from jinja2.compiler import CodeGenerator
+from jinja2.environment import Environment, Template
+from jinja2.utils import concat, escape
+
+
+def native_concat(nodes):
+    """Return a native Python type from the list of compiled nodes. If the
+    result is a single node, its value is returned. Otherwise, the nodes are
+    concatenated as strings. If the result can be parsed with
+    :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
+    string is returned.
+    """
+    head = list(islice(nodes, 2))
+
+    if not head:
+        return None
+
+    if len(head) == 1:
+        out = head[0]
+    else:
+        out = u''.join([text_type(v) for v in chain(head, nodes)])
+
+    try:
+        return literal_eval(out)
+    except (ValueError, SyntaxError, MemoryError):
+        return out
+
+
+class NativeCodeGenerator(CodeGenerator):
+    """A code generator which avoids injecting ``to_string()`` calls around the
+    internal code Jinja uses to render templates.
+    """
+
+    def visit_Output(self, node, frame):
+        """Same as :meth:`CodeGenerator.visit_Output`, but do not call
+        ``to_string`` on output nodes in generated code.
+        """
+        if self.has_known_extends and frame.require_output_check:
+            return
+
+        finalize = self.environment.finalize
+        finalize_context = getattr(finalize, 'contextfunction', False)
+        finalize_eval = getattr(finalize, 'evalcontextfunction', False)
+        finalize_env = getattr(finalize, 'environmentfunction', False)
+
+        if finalize is not None:
+            if finalize_context or finalize_eval:
+                const_finalize = None
+            elif finalize_env:
+                def const_finalize(x):
+                    return finalize(self.environment, x)
+            else:
+                const_finalize = finalize
+        else:
+            def const_finalize(x):
+                return x
+
+        # If we are inside a frame that requires output checking, we do so.
+        outdent_later = False
+
+        if frame.require_output_check:
+            self.writeline('if parent_template is None:')
+            self.indent()
+            outdent_later = True
+
+        # Try to evaluate as many chunks as possible into a static string at
+        # compile time.
+        body = []
+
+        for child in node.nodes:
+            try:
+                if const_finalize is None:
+                    raise nodes.Impossible()
+
+                const = child.as_const(frame.eval_ctx)
+            except nodes.Impossible:
+                body.append(child)
+                continue
+
+            # the frame can't be volatile here, because otherwise the as_const
+            # function would raise an Impossible exception at that point
+            try:
+                if frame.eval_ctx.autoescape:
+                    if hasattr(const, '__html__'):
+                        const = const.__html__()
+                    else:
+                        const = escape(const)
+
+                const = const_finalize(const)
+            except Exception:
+                # if something goes wrong here we evaluate the node at runtime
+                # for easier debugging
+                body.append(child)
+                continue
+
+            if body and isinstance(body[-1], list):
+                body[-1].append(const)
+            else:
+                body.append([const])
+
+        # if we have less than 3 nodes or a buffer we yield or extend/append
+        if len(body) < 3 or frame.buffer is not None:
+            if frame.buffer is not None:
+                # for one item we append, for more we extend
+                if len(body) == 1:
+                    self.writeline('%s.append(' % frame.buffer)
+                else:
+                    self.writeline('%s.extend((' % frame.buffer)
+
+                self.indent()
+
+            for item in body:
+                if isinstance(item, list):
+                    val = repr(native_concat(item))
+
+                    if frame.buffer is None:
+                        self.writeline('yield ' + val)
+                    else:
+                        self.writeline(val + ',')
+                else:
+                    if frame.buffer is None:
+                        self.writeline('yield ', item)
+                    else:
+                        self.newline(item)
+
+                    close = 0
+
+                    if finalize is not None:
+                        self.write('environment.finalize(')
+
+                        if finalize_context:
+                            self.write('context, ')
+
+                        close += 1
+
+                    self.visit(item, frame)
+
+                    if close > 0:
+                        self.write(')' * close)
+
+                    if frame.buffer is not None:
+                        self.write(',')
+
+            if frame.buffer is not None:
+                # close the open parentheses
+                self.outdent()
+                self.writeline(len(body) == 1 and ')' or '))')
+
+        # otherwise we create a format string as this is faster in that case
+        else:
+            format = []
+            arguments = []
+
+            for item in body:
+                if isinstance(item, list):
+                    format.append(native_concat(item).replace('%', '%%'))
+                else:
+                    format.append('%s')
+                    arguments.append(item)
+
+            self.writeline('yield ')
+            self.write(repr(concat(format)) + ' % (')
+            self.indent()
+
+            for argument in arguments:
+                self.newline(argument)
+                close = 0
+
+                if finalize is not None:
+                    self.write('environment.finalize(')
+
+                    if finalize_context:
+                        self.write('context, ')
+                    elif finalize_eval:
+                        self.write('context.eval_ctx, ')
+                    elif finalize_env:
+                        self.write('environment, ')
+
+                    close += 1
+
+                self.visit(argument, frame)
+                self.write(')' * close + ', ')
+
+            self.outdent()
+            self.writeline(')')
+
+        if outdent_later:
+            self.outdent()
+
+
+class NativeTemplate(Template):
+    def render(self, *args, **kwargs):
+        """Render the template to produce a native Python type. If the result
+        is a single node, its value is returned. Otherwise, the nodes are
+        concatenated as strings. If the result can be parsed with
+        :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the
+        string is returned.
+        """
+        vars = dict(*args, **kwargs)
+
+        try:
+            return native_concat(self.root_render_func(self.new_context(vars)))
+        except Exception:
+            exc_info = sys.exc_info()
+
+        return self.environment.handle_exception(exc_info, True)
+
+
+class NativeEnvironment(Environment):
+    """An environment that renders templates to native Python types."""
+
+    code_generator_class = NativeCodeGenerator
+    template_class = NativeTemplate
diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py
new file mode 100644
index 0000000..d295466
--- /dev/null
+++ b/tests/test_nativetypes.py
@@ -0,0 +1,94 @@
+import pytest
+
+from jinja2._compat import text_type
+from jinja2.exceptions import UndefinedError
+from jinja2.nativetypes import NativeEnvironment
+from jinja2.runtime import Undefined
+
+
+@pytest.fixture
+def env():
+    return NativeEnvironment()
+
+
+class TestNativeEnvironment(object):
+    def test_is_defined_native_return(self, env):
+        t = env.from_string('{{ missing is defined }}')
+        assert not t.render()
+
+    def test_undefined_native_return(self, env):
+        t = env.from_string('{{ missing }}')
+        assert isinstance(t.render(), Undefined)
+
+    def test_adding_undefined_native_return(self, env):
+        t = env.from_string('{{ 3 + missing }}')
+
+        with pytest.raises(UndefinedError):
+            t.render()
+
+    def test_cast_int(self, env):
+        t = env.from_string("{{ anumber|int }}")
+        result = t.render(anumber='3')
+        assert isinstance(result, int)
+        assert result == 3
+
+    def test_list_add(self, env):
+        t = env.from_string("{{ listone + listtwo }}")
+        result = t.render(listone=['a', 'b'], listtwo=['c', 'd'])
+        assert isinstance(result, list)
+        assert result == ['a', 'b', 'c', 'd']
+
+    def test_multi_expression_add(self, env):
+        t = env.from_string("{{ listone }} + {{ listtwo }}")
+        result = t.render(listone=['a', 'b'], listtwo=['c', 'd'])
+        assert not isinstance(result, list)
+        assert result == "['a', 'b'] + ['c', 'd']"
+
+    def test_loops(self, env):
+        t = env.from_string("{% for x in listone %}{{ x }}{% endfor %}")
+        result = t.render(listone=['a', 'b', 'c', 'd'])
+        assert isinstance(result, text_type)
+        assert result == 'abcd'
+
+    def test_loops_with_ints(self, env):
+        t = env.from_string("{% for x in listone %}{{ x }}{% endfor %}")
+        result = t.render(listone=[1, 2, 3, 4])
+        assert isinstance(result, int)
+        assert result == 1234
+
+    def test_loop_look_alike(self, env):
+        t = env.from_string("{% for x in listone %}{{ x }}{% endfor %}")
+        result = t.render(listone=[1])
+        assert isinstance(result, int)
+        assert result == 1
+
+    def test_booleans(self, env):
+        t = env.from_string("{{ boolval }}")
+        result = t.render(boolval=True)
+        assert isinstance(result, bool)
+        assert result is True
+
+        t = env.from_string("{{ boolval }}")
+        result = t.render(boolval=False)
+        assert isinstance(result, bool)
+        assert result is False
+
+        t = env.from_string("{{ 1 == 1 }}")
+        result = t.render()
+        assert isinstance(result, bool)
+        assert result is True
+
+        t = env.from_string("{{ 2 + 2 == 5 }}")
+        result = t.render()
+        assert isinstance(result, bool)
+        assert result is False
+
+        t = env.from_string("{{ None == None }}")
+        result = t.render()
+        assert isinstance(result, bool)
+        assert result is True
+
+        t = env.from_string("{{ '' == None }}")
+        result = t.render()
+        assert isinstance(result, bool)
+        assert result is False