Merge pull request #1110 from pallets/patch-traceback

rewrite traceback rewriting support

6 files changed, 234 insertions, 379 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index f572e9a..037dfaa 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -81,6 +81,10 @@ Unreleased
     the result follows Python's behavior of returning ``False`` if any
     comparison returns ``False``, rather than only the last one.
     :issue:`1102`
+-   Tracebacks for exceptions in templates show the correct line numbers
+    and source for Python >= 3.7. :issue:`1104`
+-   Tracebacks for template syntax errors in Python 3 no longer show
+    internal compiler frames. :issue:`763`
 
 
 Version 2.10.3
diff --git a/jinja2/asyncsupport.py b/jinja2/asyncsupport.py
index 7d457e3..d225962 100644
--- a/jinja2/asyncsupport.py
+++ b/jinja2/asyncsupport.py
@@ -11,7 +11,6 @@
 """
 import asyncio
 import inspect
-import sys
 from functools import update_wrapper
 
 from jinja2.environment import TemplateModule
@@ -37,10 +36,7 @@ async def generate_async(self, *args, **kwargs):
         async for event in self.root_render_func(self.new_context(vars)):
             yield event
     except Exception:
-        exc_info = sys.exc_info()
-    else:
-        return
-    yield self.environment.handle_exception(exc_info, True)
+        yield self.environment.handle_exception()
 
 
 def wrap_generate_func(original_generate):
@@ -69,8 +65,7 @@ async def render_async(self, *args, **kwargs):
     try:
         return await concat_async(self.root_render_func(ctx))
     except Exception:
-        exc_info = sys.exc_info()
-    return self.environment.handle_exception(exc_info, True)
+        return self.environment.handle_exception()
 
 
 def wrap_render_func(original_render):
diff --git a/jinja2/debug.py b/jinja2/debug.py
index d3c1a3a..1887fcf 100644
--- a/jinja2/debug.py
+++ b/jinja2/debug.py
@@ -1,378 +1,268 @@
-# -*- coding: utf-8 -*-
-"""
-    jinja2.debug
-    ~~~~~~~~~~~~
-
-    Implements the debug interface for Jinja.  This module does some pretty
-    ugly stuff with the Python traceback system in order to achieve tracebacks
-    with correct line numbers, locals and contents.
-
-    :copyright: (c) 2017 by the Jinja Team.
-    :license: BSD, see LICENSE for more details.
-"""
 import sys
-import traceback
-from types import TracebackType, CodeType
-from jinja2.utils import missing, internal_code
-from jinja2.exceptions import TemplateSyntaxError
-from jinja2._compat import iteritems, reraise, PY2
+from types import CodeType
 
-# on pypy we can take advantage of transparent proxies
-try:
-    from __pypy__ import tproxy
-except ImportError:
-    tproxy = None
+from jinja2 import TemplateSyntaxError
+from jinja2._compat import PYPY
+from jinja2.utils import internal_code
+from jinja2.utils import missing
 
 
-# how does the raise helper look like?
-try:
-    exec("raise TypeError, 'foo'")
-except SyntaxError:
-    raise_helper = 'raise __jinja_exception__[1]'
-except TypeError:
-    raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]'
+def rewrite_traceback_stack(source=None):
+    """Rewrite the current exception to replace any tracebacks from
+    within compiled template code with tracebacks that look like they
+    came from the template source.
 
+    This must be called within an ``except`` block.
 
-class TracebackFrameProxy(object):
-    """Proxies a traceback frame."""
+    :param exc_info: A :meth:`sys.exc_info` tuple. If not provided,
+        the current ``exc_info`` is used.
+    :param source: For ``TemplateSyntaxError``, the original source if
+        known.
+    :return: A :meth:`sys.exc_info` tuple that can be re-raised.
+    """
+    exc_type, exc_value, tb = sys.exc_info()
+    # The new stack of traceback objects, to be joined together by
+    # tb_set_next later.
+    stack = []
 
-    def __init__(self, tb):
-        self.tb = tb
-        self._tb_next = None
+    if isinstance(exc_value, TemplateSyntaxError):
+        exc_value.source = source
+        # The exception doesn't need to output location info manually.
+        exc_value.translated = True
 
-    @property
-    def tb_next(self):
-        return self._tb_next
+        try:
+            # Remove the old traceback on Python 3, otherwise the frames
+            # from the compiler still show up.
+            exc_value.with_traceback(None)
+        except AttributeError:
+            pass
 
-    def set_next(self, next):
-        if tb_set_next is not None:
-            try:
-                tb_set_next(self.tb, next and next.tb or None)
-            except Exception:
-                # this function can fail due to all the hackery it does
-                # on various python implementations.  We just catch errors
-                # down and ignore them if necessary.
-                pass
-        self._tb_next = next
-
-    @property
-    def is_jinja_frame(self):
-        return '__jinja_template__' in self.tb.tb_frame.f_globals
-
-    def __getattr__(self, name):
-        return getattr(self.tb, name)
-
-
-def make_frame_proxy(frame):
-    proxy = TracebackFrameProxy(frame)
-    if tproxy is None:
-        return proxy
-    def operation_handler(operation, *args, **kwargs):
-        if operation in ('__getattribute__', '__getattr__'):
-            return getattr(proxy, args[0])
-        elif operation == '__setattr__':
-            proxy.__setattr__(*args, **kwargs)
-        else:
-            return getattr(proxy, operation)(*args, **kwargs)
-    return tproxy(TracebackType, operation_handler)
-
-
-class ProcessedTraceback(object):
-    """Holds a Jinja preprocessed traceback for printing or reraising."""
-
-    def __init__(self, exc_type, exc_value, frames):
-        assert frames, 'no frames for this traceback?'
-        self.exc_type = exc_type
-        self.exc_value = exc_value
-        self.frames = frames
-
-        # newly concatenate the frames (which are proxies)
-        prev_tb = None
-        for tb in self.frames:
-            if prev_tb is not None:
-                prev_tb.set_next(tb)
-            prev_tb = tb
-        prev_tb.set_next(None)
-
-    def render_as_text(self, limit=None):
-        """Return a string with the traceback."""
-        lines = traceback.format_exception(self.exc_type, self.exc_value,
-                                           self.frames[0], limit=limit)
-        return ''.join(lines).rstrip()
-
-    def render_as_html(self, full=False):
-        """Return a unicode string with the traceback as rendered HTML."""
-        from jinja2.debugrenderer import render_traceback
-        return u'%s\n\n<!--\n%s\n-->' % (
-            render_traceback(self, full=full),
-            self.render_as_text().decode('utf-8', 'replace')
+        # Outside of runtime, so the frame isn't executing template
+        # code, but it still needs to point at the template.
+        tb = fake_traceback(
+            exc_value, None, exc_value.filename or "<unknown>", exc_value.lineno
         )
-
-    @property
-    def is_template_syntax_error(self):
-        """`True` if this is a template syntax error."""
-        return isinstance(self.exc_value, TemplateSyntaxError)
-
-    @property
-    def exc_info(self):
-        """Exception info tuple with a proxy around the frame objects."""
-        return self.exc_type, self.exc_value, self.frames[0]
-
-    @property
-    def standard_exc_info(self):
-        """Standard python exc_info for re-raising"""
-        tb = self.frames[0]
-        # the frame will be an actual traceback (or transparent proxy) if
-        # we are on pypy or a python implementation with support for tproxy
-        if type(tb) is not TracebackType:
-            tb = tb.tb
-        return self.exc_type, self.exc_value, tb
-
-
-def make_traceback(exc_info, source_hint=None):
-    """Creates a processed traceback object from the exc_info."""
-    exc_type, exc_value, tb = exc_info
-    if isinstance(exc_value, TemplateSyntaxError):
-        exc_info = translate_syntax_error(exc_value, source_hint)
-        initial_skip = 0
     else:
-        initial_skip = 1
-    return translate_exception(exc_info, initial_skip)
-
-
-def translate_syntax_error(error, source=None):
-    """Rewrites a syntax error to please traceback systems."""
-    error.source = source
-    error.translated = True
-    exc_info = (error.__class__, error, None)
-    filename = error.filename
-    if filename is None:
-        filename = '<unknown>'
-    return fake_exc_info(exc_info, filename, error.lineno)
-
-
-def translate_exception(exc_info, initial_skip=0):
-    """If passed an exc_info it will automatically rewrite the exceptions
-    all the way down to the correct line numbers and frames.
-    """
-    tb = exc_info[2]
-    frames = []
-
-    # skip some internal frames if wanted
-    for x in range(initial_skip):
-        if tb is not None:
-            tb = tb.tb_next
-    initial_tb = tb
+        # Skip the frame for the render function.
+        tb = tb.tb_next
 
+    # Build the stack of traceback object, replacing any in template
+    # code with the source file and line information.
     while tb is not None:
-        # skip frames decorated with @internalcode.  These are internal
-        # calls we can't avoid and that are useless in template debugging
-        # output.
+        # Skip frames decorated with @internalcode. These are internal
+        # calls that aren't useful in template debugging output.
         if tb.tb_frame.f_code in internal_code:
             tb = tb.tb_next
             continue
 
-        # save a reference to the next frame if we override the current
-        # one with a faked one.
-        next = tb.tb_next
+        template = tb.tb_frame.f_globals.get("__jinja_template__")
 
-        # fake template exceptions
-        template = tb.tb_frame.f_globals.get('__jinja_template__')
         if template is not None:
             lineno = template.get_corresponding_lineno(tb.tb_lineno)
-            tb = fake_exc_info(exc_info[:2] + (tb,), template.filename,
-                               lineno)[2]
-
-        frames.append(make_frame_proxy(tb))
-        tb = next
-
-    # if we don't have any exceptions in the frames left, we have to
-    # reraise it unchanged.
-    # XXX: can we backup here?  when could this happen?
-    if not frames:
-        reraise(exc_info[0], exc_info[1], exc_info[2])
-
-    return ProcessedTraceback(exc_info[0], exc_info[1], frames)
-
-
-def get_jinja_locals(real_locals):
-    ctx = real_locals.get('context')
-    if ctx:
-        locals = ctx.get_all().copy()
-    else:
-        locals = {}
+            fake_tb = fake_traceback(exc_value, tb, template.filename, lineno)
+            stack.append(fake_tb)
+        else:
+            stack.append(tb)
 
-    local_overrides = {}
+        tb = tb.tb_next
 
-    for name, value in iteritems(real_locals):
-        if not name.startswith('l_') or value is missing:
-            continue
-        try:
-            _, depth, name = name.split('_', 2)
-            depth = int(depth)
-        except ValueError:
-            continue
-        cur_depth = local_overrides.get(name, (-1,))[0]
-        if cur_depth < depth:
-            local_overrides[name] = (depth, value)
+    tb_next = None
 
-    for name, (_, value) in iteritems(local_overrides):
-        if value is missing:
-            locals.pop(name, None)
-        else:
-            locals[name] = value
+    # Assign tb_next in reverse to avoid circular references.
+    for tb in reversed(stack):
+        tb_next = tb_set_next(tb, tb_next)
 
-    return locals
+    return exc_type, exc_value, tb_next
 
 
-def fake_exc_info(exc_info, filename, lineno):
-    """Helper for `translate_exception`."""
-    exc_type, exc_value, tb = exc_info
+def fake_traceback(exc_value, tb, filename, lineno):
+    """Produce a new traceback object that looks like it came from the
+    template source instead of the compiled code. The filename, line
+    number, and location name will point to the template, and the local
+    variables will be the current template context.
 
-    # figure the real context out
+    :param exc_value: The original exception to be re-raised to create
+        the new traceback.
+    :param tb: The original traceback to get the local variables and
+        code info from.
+    :param filename: The template filename.
+    :param lineno: The line number in the template source.
+    """
     if tb is not None:
-        locals = get_jinja_locals(tb.tb_frame.f_locals)
-
-        # if there is a local called __jinja_exception__, we get
-        # rid of it to not break the debug functionality.
-        locals.pop('__jinja_exception__', None)
+        # Replace the real locals with the context that would be
+        # available at that point in the template.
+        locals = get_template_locals(tb.tb_frame.f_locals)
+        locals.pop("__jinja_exception__", None)
     else:
         locals = {}
 
-    # assamble fake globals we need
     globals = {
-        '__name__':             filename,
-        '__file__':             filename,
-        '__jinja_exception__':  exc_info[:2],
-
-        # we don't want to keep the reference to the template around
-        # to not cause circular dependencies, but we mark it as Jinja
-        # frame for the ProcessedTraceback
-        '__jinja_template__':   None
+        "__name__": filename,
+        "__file__": filename,
+        "__jinja_exception__": exc_value,
     }
+    # Raise an exception at the correct line number.
+    code = compile('\n' * (lineno - 1) + "raise __jinja_exception__", filename, "exec")
 
-    # and fake the exception
-    code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec')
-
-    # if it's possible, change the name of the code.  This won't work
-    # on some python environments such as google appengine
+    # Build a new code object that points to the template file and
+    # replaces the location with a block name.
     try:
-        if tb is None:
-            location = 'template'
-        else:
+        location = "template"
+
+        if tb is not None:
             function = tb.tb_frame.f_code.co_name
-            if function == 'root':
-                location = 'top-level template code'
-            elif function.startswith('block_'):
+
+            if function == "root":
+                location = "top-level template code"
+            elif function.startswith("block_"):
                 location = 'block "%s"' % function[6:]
-            else:
-                location = 'template'
-
-        if PY2:
-            code = CodeType(0, code.co_nlocals, code.co_stacksize,
-                            code.co_flags, code.co_code, code.co_consts,
-                            code.co_names, code.co_varnames, filename,
-                            location, code.co_firstlineno,
-                            code.co_lnotab, (), ())
-        else:
-            code = CodeType(0, code.co_kwonlyargcount,
-                            code.co_nlocals, code.co_stacksize,
-                            code.co_flags, code.co_code, code.co_consts,
-                            code.co_names, code.co_varnames, filename,
-                            location, code.co_firstlineno,
-                            code.co_lnotab, (), ())
-    except Exception as e:
+
+        # Collect arguments for the new code object. CodeType only
+        # accepts positional arguments, and arguments were inserted in
+        # new Python versions.
+        code_args = []
+
+        for attr in (
+            "argcount",
+            "posonlyargcount",  # Python 3.8
+            "kwonlyargcount",  # Python 3
+            "nlocals",
+            "stacksize",
+            "flags",
+            "code",  # codestring
+            "consts",  # constants
+            "names",
+            "varnames",
+            ("filename", filename),
+            ("name", location),
+            "firstlineno",
+            "lnotab",
+            "freevars",
+            "cellvars",
+        ):
+            if isinstance(attr, tuple):
+                # Replace with given value.
+                code_args.append(attr[1])
+                continue
+
+            try:
+                # Copy original value if it exists.
+                code_args.append(getattr(code, "co_" + attr))
+            except AttributeError:
+                # Some arguments were added later.
+                continue
+
+        code = CodeType(*code_args)
+    except Exception:
+        # Some environments such as Google App Engine don't support
+        # modifying code objects.
         pass
 
-    # execute the code and catch the new traceback
+    # Execute the new code, which is guaranteed to raise, and return
+    # the new traceback without this frame.
     try:
         exec(code, globals, locals)
     except:
-        exc_info = sys.exc_info()
-        new_tb = exc_info[2].tb_next
-
-    # return without this frame
-    return exc_info[:2] + (new_tb,)
+        return sys.exc_info()[2].tb_next
 
 
-def _init_ugly_crap():
-    """This function implements a few ugly things so that we can patch the
-    traceback objects.  The function returned allows resetting `tb_next` on
-    any python traceback object.  Do not attempt to use this on non cpython
-    interpreters
+def get_template_locals(real_locals):
+    """Based on the runtime locals, get the context that would be
+    available at that point in the template.
     """
-    import ctypes
-    from types import TracebackType
+    # Start with the current template context.
+    ctx = real_locals.get("context")
 
-    if PY2:
-        # figure out size of _Py_ssize_t for Python 2:
-        if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
-            _Py_ssize_t = ctypes.c_int64
-        else:
-            _Py_ssize_t = ctypes.c_int
+    if ctx:
+        data = ctx.get_all().copy()
     else:
-        # platform ssize_t on Python 3
-        _Py_ssize_t = ctypes.c_ssize_t
+        data = {}
 
-    # regular python
-    class _PyObject(ctypes.Structure):
-        pass
-    _PyObject._fields_ = [
-        ('ob_refcnt', _Py_ssize_t),
-        ('ob_type', ctypes.POINTER(_PyObject))
-    ]
-
-    # python with trace
-    if hasattr(sys, 'getobjects'):
-        class _PyObject(ctypes.Structure):
-            pass
-        _PyObject._fields_ = [
-            ('_ob_next', ctypes.POINTER(_PyObject)),
-            ('_ob_prev', ctypes.POINTER(_PyObject)),
-            ('ob_refcnt', _Py_ssize_t),
-            ('ob_type', ctypes.POINTER(_PyObject))
-        ]
+    # Might be in a derived context that only sets local variables
+    # rather than pushing a context. Local variables follow the scheme
+    # l_depth_name. Find the highest-depth local that has a value for
+    # each name.
+    local_overrides = {}
 
-    class _Traceback(_PyObject):
-        pass
-    _Traceback._fields_ = [
-        ('tb_next', ctypes.POINTER(_Traceback)),
-        ('tb_frame', ctypes.POINTER(_PyObject)),
-        ('tb_lasti', ctypes.c_int),
-        ('tb_lineno', ctypes.c_int)
-    ]
-
-    def tb_set_next(tb, next):
-        """Set the tb_next attribute of a traceback object."""
-        if not (isinstance(tb, TracebackType) and
-                (next is None or isinstance(next, TracebackType))):
-            raise TypeError('tb_set_next arguments must be traceback objects')
-        obj = _Traceback.from_address(id(tb))
-        if tb.tb_next is not None:
-            old = _Traceback.from_address(id(tb.tb_next))
-            old.ob_refcnt -= 1
-        if next is None:
-            obj.tb_next = ctypes.POINTER(_Traceback)()
+    for name, value in real_locals.items():
+        if not name.startswith("l_") or value is missing:
+            # Not a template variable, or no longer relevant.
+            continue
+
+        try:
+            _, depth, name = name.split("_", 2)
+            depth = int(depth)
+        except ValueError:
+            continue
+
+        cur_depth = local_overrides.get(name, (-1,))[0]
+
+        if cur_depth < depth:
+            local_overrides[name] = (depth, value)
+
+    # Modify the context with any derived context.
+    for name, (_, value) in local_overrides.items():
+        if value is missing:
+            data.pop(name, None)
         else:
-            next = _Traceback.from_address(id(next))
-            next.ob_refcnt += 1
-            obj.tb_next = ctypes.pointer(next)
+            data[name] = value
 
-    return tb_set_next
+    return data
 
 
-# try to get a tb_set_next implementation if we don't have transparent
-# proxies.
-tb_set_next = None
-if tproxy is None:
-    # traceback.tb_next can be modified since CPython 3.7
-    if sys.version_info >= (3, 7):
-        def tb_set_next(tb, next):
-            tb.tb_next = next
+if sys.version_info >= (3, 7):
+    # tb_next is directly assignable as of Python 3.7
+    def tb_set_next(tb, tb_next):
+        tb.tb_next = tb_next
+        return tb
+elif PYPY:
+    # PyPy might have special support, and won't work with ctypes.
+    try:
+        import tputil
+    except ImportError:
+        # Without tproxy support, use the original traceback.
+        def tb_set_next(tb, tb_next):
+            return tb
     else:
-        # On Python 3.6 and older, use ctypes
-        try:
-            tb_set_next = _init_ugly_crap()
-        except Exception:
-            pass
-del _init_ugly_crap
+        # With tproxy support, create a proxy around the traceback that
+        # returns the new tb_next.
+        def tb_set_next(tb, tb_next):
+            def controller(op):
+                if op.opname == "__getattribute__" and op.args[0] == "tb_next":
+                    return tb_next
+
+                return op.delegate()
+
+            return tputil.make_proxy(controller, obj=tb)
+else:
+    # Use ctypes to assign tb_next at the C level since it's read-only
+    # from Python.
+    import ctypes
+
+    class _CTraceback(ctypes.Structure):
+        _fields_ = [
+            # Extra PyObject slots when compiled with Py_TRACE_REFS.
+            (
+                "PyObject_HEAD",
+                ctypes.c_byte * (32 if hasattr(sys, "getobjects") else 16),
+            ),
+            # Only care about tb_next as an object, not a traceback.
+            ("tb_next", ctypes.py_object),
+        ]
+
+    def tb_set_next(tb, tb_next):
+        c_tb = _CTraceback.from_address(id(tb))
+
+        # Clear out the old tb_next.
+        if tb.tb_next is not None:
+            c_tb_next = ctypes.py_object(tb.tb_next)
+            c_tb.tb_next = ctypes.py_object()
+            ctypes.pythonapi.Py_DecRef(c_tb_next)
+
+        # Assign the new tb_next.
+        if tb_next is not None:
+            c_tb_next = ctypes.py_object(tb_next)
+            ctypes.pythonapi.Py_IncRef(c_tb_next)
+            c_tb.tb_next = c_tb_next
+
+        return tb
diff --git a/jinja2/environment.py b/jinja2/environment.py
index 974479f..2bfc018 100644
--- a/jinja2/environment.py
+++ b/jinja2/environment.py
@@ -36,10 +36,6 @@ from jinja2._compat import imap, ifilter, string_types, iteritems, \
 # for direct template usage we have up to ten living environments
 _spontaneous_environments = LRUCache(10)
 
-# the function to create jinja traceback objects.  This is dynamically
-# imported on the first exception in the exception handler.
-_make_traceback = None
-
 
 def get_spontaneous_environment(cls, *args):
     """Return a new spontaneous environment. A spontaneous environment
@@ -251,10 +247,6 @@ class Environment(object):
     #: must not be modified
     shared = False
 
-    #: these are currently EXPERIMENTAL undocumented features.
-    exception_handler = None
-    exception_formatter = None
-
     #: the class that is used for code generation.  See
     #: :class:`~jinja2.compiler.CodeGenerator` for more information.
     code_generator_class = CodeGenerator
@@ -493,8 +485,7 @@ class Environment(object):
         try:
             return self._parse(source, name, filename)
         except TemplateSyntaxError:
-            exc_info = sys.exc_info()
-        self.handle_exception(exc_info, source_hint=source)
+            self.handle_exception(source=source)
 
     def _parse(self, source, name, filename):
         """Internal parsing function used by `parse` and `compile`."""
@@ -514,8 +505,7 @@ class Environment(object):
         try:
             return self.lexer.tokeniter(source, name, filename)
         except TemplateSyntaxError:
-            exc_info = sys.exc_info()
-        self.handle_exception(exc_info, source_hint=source)
+            self.handle_exception(source=source)
 
     def preprocess(self, source, name=None, filename=None):
         """Preprocesses the source with all extensions.  This is automatically
@@ -591,8 +581,7 @@ class Environment(object):
                 filename = encode_filename(filename)
             return self._compile(source, filename)
         except TemplateSyntaxError:
-            exc_info = sys.exc_info()
-        self.handle_exception(exc_info, source_hint=source_hint)
+            self.handle_exception(source=source_hint)
 
     def compile_expression(self, source, undefined_to_none=True):
         """A handy helper method that returns a callable that accepts keyword
@@ -623,7 +612,6 @@ class Environment(object):
         .. versionadded:: 2.1
         """
         parser = Parser(self, source, state='variable')
-        exc_info = None
         try:
             expr = parser.parse_expression()
             if not parser.stream.eos:
@@ -632,9 +620,9 @@ class Environment(object):
                                           None, None)
             expr.set_environment(self)
         except TemplateSyntaxError:
-            exc_info = sys.exc_info()
-        if exc_info is not None:
-            self.handle_exception(exc_info, source_hint=source)
+            if sys.exc_info() is not None:
+                self.handle_exception(source=source)
+
         body = [nodes.Assign(nodes.Name('result', 'store'), expr, lineno=1)]
         template = self.from_string(nodes.Template(body, lineno=1))
         return TemplateExpression(template, undefined_to_none)
@@ -761,27 +749,12 @@ class Environment(object):
             x = list(ifilter(filter_func, x))
         return x
 
-    def handle_exception(self, exc_info=None, rendered=False, source_hint=None):
+    def handle_exception(self, source=None):
         """Exception handling helper.  This is used internally to either raise
         rewritten exceptions or return a rendered traceback for the template.
         """
-        global _make_traceback
-        if exc_info is None:
-            exc_info = sys.exc_info()
-
-        # the debugging module is imported when it's used for the first time.
-        # we're doing a lot of stuff there and for applications that do not
-        # get any exceptions in template rendering there is no need to load
-        # all of that.
-        if _make_traceback is None:
-            from jinja2.debug import make_traceback as _make_traceback
-        traceback = _make_traceback(exc_info, source_hint)
-        if rendered and self.exception_formatter is not None:
-            return self.exception_formatter(traceback)
-        if self.exception_handler is not None:
-            self.exception_handler(traceback)
-        exc_type, exc_value, tb = traceback.standard_exc_info
-        reraise(exc_type, exc_value, tb)
+        from jinja2.debug import rewrite_traceback_stack
+        reraise(*rewrite_traceback_stack(source=source))
 
     def join_path(self, template, parent):
         """Join a template with the parent.  By default all the lookups are
@@ -1013,8 +986,7 @@ class Template(object):
         try:
             return concat(self.root_render_func(self.new_context(vars)))
         except Exception:
-            exc_info = sys.exc_info()
-        return self.environment.handle_exception(exc_info, True)
+            self.environment.handle_exception()
 
     def render_async(self, *args, **kwargs):
         """This works similar to :meth:`render` but returns a coroutine
@@ -1048,10 +1020,7 @@ class Template(object):
             for event in self.root_render_func(self.new_context(vars)):
                 yield event
         except Exception:
-            exc_info = sys.exc_info()
-        else:
-            return
-        yield self.environment.handle_exception(exc_info, True)
+            yield self.environment.handle_exception()
 
     def generate_async(self, *args, **kwargs):
         """An async version of :meth:`generate`.  Works very similarly but
diff --git a/jinja2/nativetypes.py b/jinja2/nativetypes.py
index 7128a33..e0de277 100644
--- a/jinja2/nativetypes.py
+++ b/jinja2/nativetypes.py
@@ -1,4 +1,3 @@
-import sys
 import types
 from ast import literal_eval
 from itertools import islice, chain
@@ -102,9 +101,7 @@ class NativeTemplate(Template):
                 self.root_render_func(self.new_context(vars)), preserve_quotes=False
             )
         except Exception:
-            exc_info = sys.exc_info()
-
-        return self.environment.handle_exception(exc_info, True)
+            return self.environment.handle_exception()
 
 
 NativeEnvironment.template_class = NativeTemplate
diff --git a/tests/test_debug.py b/tests/test_debug.py
index 9a6fbb5..9e25fbd 100644
--- a/tests/test_debug.py
+++ b/tests/test_debug.py
@@ -71,9 +71,9 @@ ZeroDivisionError: (int(eger)? )?division (or modulo )?by zero
   line 42''')
 
     def test_local_extraction(self):
-        from jinja2.debug import get_jinja_locals
+        from jinja2.debug import get_template_locals
         from jinja2.runtime import missing
-        locals = get_jinja_locals({
+        locals = get_template_locals({
             'l_0_foo': 42,
             'l_1_foo': 23,
             'l_2_foo': 13,