Merge pull request #1072 from pallets/autoescape-docs2.10.x

explain what autoescaping does

19 files changed, 130 insertions, 123 deletions

diff --git a/docs/api.rst b/docs/api.rst
index 3f56e5f..7faf08a 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -36,8 +36,8 @@ This will create a template environment with the default settings and a
 loader that looks up the templates in the `templates` folder inside the
 `yourapplication` python package.  Different loaders are available
 and you can also write your own if you want to load templates from a
-database or other resources.  This also enables autoescaping for HTML and
-XML files.
+database or other resources.  This also enables :ref:`autoescaping` for
+HTML and XML files.
 
 To load a template from this environment you just have to call the
 :meth:`get_template` method which then returns the loaded :class:`Template`::
@@ -52,12 +52,6 @@ Using a template loader rather than passing strings to :class:`Template`
 or :meth:`Environment.from_string` has multiple advantages.  Besides being
 a lot easier to use it also enables template inheritance.
 
-.. admonition:: Notes on Autoescaping
-
-   In future versions of Jinja2 we might enable autoescaping by default
-   for security reasons.  As such you are encouraged to explicitly
-   configure autoescaping now instead of relying on the default.
-
 
 Unicode
 -------
@@ -254,44 +248,37 @@ useful if you want to dig deeper into Jinja2 or :ref:`develop extensions
     :members: disable_buffering, enable_buffering, dump
 
 
+.. _autoescaping:
+
 Autoescaping
 ------------
 
-.. versionchanged:: 2.4
-
-Jinja2 now comes with autoescaping support.  As of Jinja 2.9 the
-autoescape extension is removed and built-in.  However autoescaping is
-not yet enabled by default though this will most likely change in the
-future.  It's recommended to configure a sensible default for
-autoescaping.  This makes it possible to enable and disable autoescaping
-on a per-template basis (HTML versus text for instance).
+Autoescaping is a feature to mitigate injection attacks when rendering
+HTML and XML documents. When autoescaping is enabled, Jinja will use
+`MarkupSafe`_ to escape unsafe characters in the output of expressions,
+unless the output is marked safe. For example, if a comment contained
+``<script>alert("hello")</script>``, the tags would be rendered with
+escapes like ``&lt;script&gt;``. In a user's browser, the comment would
+display as text, rather than being interpreted as a script.
+
+Because Jinja can be used to render any type of document for many types
+of applications, not just HTML with untrusted input, autoescaping is not
+enabled by default. You should configure a sensible default based on
+your use case when creating the environment. The
+:func:`~jinja2.select_autoescape` function can be used to enable
+autoescaping for HTML templates while disabling it in text templates.
 
 .. autofunction:: jinja2.select_autoescape
 
-Here a recommended setup that enables autoescaping for templates ending
-in ``'.html'``, ``'.htm'`` and ``'.xml'`` and disabling it by default
-for all other extensions.  You can use the :func:`~jinja2.select_autoescape`
-function for this::
+You can also pass ``autoescape=True`` to enable it unconditionally, or
+pass your own function that takes the name of the template and returns
+whether it should be enabled. When writing a function, make sure to
+accept ``None`` as a name, as that will be passed for string templates.
 
-    from jinja2 import Environment, select_autoescape
-    env = Environment(autoescape=select_autoescape(['html', 'htm', 'xml']),
-                      loader=PackageLoader('mypackage'))
+Inside a template the behaviour can be temporarily changed by using
+the ``autoescape`` block, see :ref:`autoescape-overrides`.
 
-The :func:`~jinja.select_autoescape` function returns a function that
-works rougly like this::
-
-    def autoescape(template_name):
-        if template_name is None:
-            return False
-        if template_name.endswith(('.html', '.htm', '.xml'))
-
-When implementing a guessing autoescape function, make sure you also
-accept `None` as valid template name.  This will be passed when generating
-templates from strings.  You should always configure autoescaping as
-defaults in the future might change.
-
-Inside the templates the behaviour can be temporarily changed by using
-the `autoescape` block (see :ref:`autoescape-overrides`).
+.. _MarkupSafe: https://markupsafe.palletsprojects.com/
 
 
 .. _identifier-naming:
@@ -637,28 +624,14 @@ functions to a Jinja2 environment.
 
 .. autofunction:: jinja2.evalcontextfunction
 
-.. function:: escape(s)
-
-    Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s`
-    to HTML-safe sequences.  Use this if you need to display text that might
-    contain such characters in HTML.  This function will not escaped objects
-    that do have an HTML representation such as already escaped data.
-
-    The return value is a :class:`Markup` string.
-
 .. autofunction:: jinja2.clear_caches
 
 .. autofunction:: jinja2.is_undefined
 
-.. autoclass:: jinja2.Markup([string])
-    :members: escape, unescape, striptags
-
-.. admonition:: Note
-
-    The Jinja2 :class:`Markup` class is compatible with at least Pylons and
-    Genshi.  It's expected that more template engines and framework will pick
-    up the `__html__` concept soon.
+.. autofunction:: markupsafe.escape
 
+.. autoclass:: markupsafe.Markup
+    :members: escape, unescape, striptags
 
 Exceptions
 ----------
@@ -738,7 +711,8 @@ paragraphs and marks the return value as safe HTML string if autoescaping is
 enabled::
 
     import re
-    from jinja2 import evalcontextfilter, Markup, escape
+    from jinja2 import evalcontextfilter
+    from markupsafe import Markup, escape
 
     _paragraph_re = re.compile(r'(?:\r\n|\r|\n){2,}')
 
diff --git a/docs/extensions.rst b/docs/extensions.rst
index 12c5890..8211858 100644
--- a/docs/extensions.rst
+++ b/docs/extensions.rst
@@ -132,7 +132,7 @@ extraction tools in case you are not using Babel's.
 
 What's the big difference between standard and newstyle gettext calls?  In
 general they are less to type and less error prone.  Also if they are used
-in an autoescaping environment they better support automatic escaping.
+in an :ref:`autescaping` environment they better support automatic escaping.
 Here are some common differences between old and new calls:
 
 standard gettext:
diff --git a/docs/faq.rst b/docs/faq.rst
index 743a6cb..73d4659 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -95,7 +95,7 @@ means that you will less likely have an XSS problem it also causes a huge
 amount of extra processing in the template engine which can cause serious
 performance problems.  As Python doesn't provide a way to mark strings as
 unsafe Jinja has to hack around that limitation by providing a custom
-string class (the :class:`Markup` string) that safely interacts with safe
+string class (the :class:`~markupsafe.Markup` string) that safely interacts with safe
 and unsafe strings.
 
 With explicit escaping however the template engine doesn't have to perform
diff --git a/docs/templates.rst b/docs/templates.rst
index 4f61469..fd3e39f 100644
--- a/docs/templates.rst
+++ b/docs/templates.rst
@@ -391,7 +391,7 @@ this template, it first locates the parent.  The extends tag should be the
 first tag in the template.  Everything before it is printed out normally and
 may cause confusion.  For details about this behavior and how to take
 advantage of it, see :ref:`null-master-fallback`. Also a block will always be
-filled in regardless of whether the surrounding condition is evaluated to be true 
+filled in regardless of whether the surrounding condition is evaluated to be true
 or false.
 
 The filename of the template depends on the template loader.  For example, the
@@ -527,7 +527,7 @@ When automatic escaping is enabled, everything is escaped by default except
 for values explicitly marked as safe.  Variables and expressions
 can be marked as safe either in:
 
-a. the context dictionary by the application with `MarkupSafe.Markup`, or
+a. the context dictionary by the application with :class:`markupsafe.Markup`, or
 b. the template, with the `|safe` filter
 
 The main problem with this approach is that Python itself doesn't have the
@@ -543,7 +543,7 @@ data that is marked as safe.
 
 String literals in templates with automatic escaping are considered unsafe
 because native Python strings (``str``, ``unicode``, ``basestring``) are not
-`MarkupSafe.Markup` strings with an ``__html__`` attribute.
+:class:`markupsafe.Markup` strings with an ``__html__`` attribute.
 
 .. _list-of-control-structures:
 
@@ -1672,8 +1672,8 @@ Autoescape Overrides
 
 .. versionadded:: 2.4
 
-If you want you can activate and deactivate the autoescaping from within
-the templates.
+If you want you can activate and deactivate :ref:`autoescaping` from within
+a template.
 
 Example::
 
@@ -1685,7 +1685,7 @@ Example::
         Autoescaping is inactive within this block
     {% endautoescape %}
 
-After an `endautoescape` the behavior is reverted to what it was before.
+After an ``endautoescape`` the behavior is reverted to what it was before.
 
 .. admonition:: Extension
 
diff --git a/jinja2/__init__.py b/jinja2/__init__.py
index 0eaf721..3ce857a 100644
--- a/jinja2/__init__.py
+++ b/jinja2/__init__.py
@@ -53,9 +53,10 @@ from jinja2.exceptions import TemplateError, UndefinedError, \
 # decorators and public utilities
 from jinja2.filters import environmentfilter, contextfilter, \
      evalcontextfilter
-from jinja2.utils import Markup, escape, clear_caches, \
+from jinja2.utils import clear_caches, \
      environmentfunction, evalcontextfunction, contextfunction, \
      is_undefined, select_autoescape
+from markupsafe import Markup, escape
 
 __all__ = [
     'Environment', 'Template', 'BaseLoader', 'FileSystemLoader',
diff --git a/jinja2/asyncsupport.py b/jinja2/asyncsupport.py
index b1e7b5c..7ceddf4 100644
--- a/jinja2/asyncsupport.py
+++ b/jinja2/asyncsupport.py
@@ -14,7 +14,9 @@ import asyncio
 import inspect
 from functools import update_wrapper
 
-from jinja2.utils import concat, internalcode, Markup
+from markupsafe import Markup
+
+from jinja2.utils import concat, internalcode
 from jinja2.environment import TemplateModule
 from jinja2.runtime import LoopContextBase, _last_iteration
 
diff --git a/jinja2/compiler.py b/jinja2/compiler.py
index d534a82..10aed13 100644
--- a/jinja2/compiler.py
+++ b/jinja2/compiler.py
@@ -9,21 +9,22 @@
     :license: BSD, see LICENSE for more details.
 """
 from itertools import chain
-from copy import deepcopy
-from keyword import iskeyword as is_python_keyword
 from functools import update_wrapper
+from keyword import iskeyword as is_python_keyword
+
+from markupsafe import Markup, escape
+
 from jinja2 import nodes
 from jinja2.nodes import EvalContext
 from jinja2.visitor import NodeVisitor
 from jinja2.optimizer import Optimizer
 from jinja2.exceptions import TemplateAssertionError
-from jinja2.utils import Markup, concat, escape
+from jinja2.utils import concat
 from jinja2._compat import range_type, text_type, string_types, \
      iteritems, NativeStringIO, imap, izip
 from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \
      VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED
 
-
 operators = {
     'eq':       '==',
     'ne':       '!=',
diff --git a/jinja2/environment.py b/jinja2/environment.py
index 549d9af..b7f8b2f 100644
--- a/jinja2/environment.py
+++ b/jinja2/environment.py
@@ -12,6 +12,9 @@ import os
 import sys
 import weakref
 from functools import reduce, partial
+
+from markupsafe import Markup
+
 from jinja2 import nodes
 from jinja2.defaults import BLOCK_START_STRING, \
      BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
@@ -26,7 +29,7 @@ from jinja2.compiler import generate, CodeGenerator
 from jinja2.runtime import Undefined, new_context, Context
 from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \
      TemplatesNotFound, TemplateRuntimeError
-from jinja2.utils import import_string, LRUCache, Markup, missing, \
+from jinja2.utils import import_string, LRUCache, missing, \
      concat, consume, internalcode, have_async_gen
 from jinja2._compat import imap, ifilter, string_types, iteritems, \
      text_type, reraise, implements_iterator, implements_to_string, \
@@ -189,15 +192,14 @@ class Environment(object):
             ``None`` implicitly into an empty string here.
 
         `autoescape`
-            If set to ``True`` the XML/HTML autoescaping feature is enabled by
-            default.  For more details about autoescaping see
-            :class:`~jinja2.utils.Markup`.  As of Jinja 2.4 this can also
-            be a callable that is passed the template name and has to
-            return ``True`` or ``False`` depending on autoescape should be
-            enabled by default.
+            If set to ``True`` the HTML/XML autoescaping feature is
+            enabled by default. For more details about autoescaping see
+            :ref:`autoescaping`. This can also be a callable that is
+            passed the template name and returns whether autoescaping
+            should be enabled.
 
             .. versionchanged:: 2.4
-               `autoescape` can now be a function
+                Can be a function.
 
         `loader`
             The template loader for this environment.
diff --git a/jinja2/ext.py b/jinja2/ext.py
index 0734a84..b70b50b 100644
--- a/jinja2/ext.py
+++ b/jinja2/ext.py
@@ -12,6 +12,8 @@
 """
 import re
 
+from markupsafe import Markup
+
 from jinja2 import nodes
 from jinja2.defaults import BLOCK_START_STRING, \
      BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
@@ -21,7 +23,7 @@ from jinja2.defaults import BLOCK_START_STRING, \
 from jinja2.environment import Environment
 from jinja2.runtime import concat
 from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
-from jinja2.utils import contextfunction, import_string, Markup
+from jinja2.utils import contextfunction, import_string
 from jinja2._compat import with_metaclass, string_types, iteritems
 
 
diff --git a/jinja2/filters.py b/jinja2/filters.py
index 267dddd..bcb651f 100644
--- a/jinja2/filters.py
+++ b/jinja2/filters.py
@@ -12,14 +12,16 @@ import re
 import math
 import random
 import warnings
-
 from itertools import groupby, chain
 from collections import namedtuple
-from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \
+
+from markupsafe import Markup, escape, soft_unicode
+
+from jinja2.utils import pformat, urlize, \
      unicode_urlencode, htmlsafe_json_dumps
 from jinja2.runtime import Undefined
 from jinja2.exceptions import FilterArgumentError
-from jinja2._compat import imap, string_types, text_type, iteritems, PY2
+from jinja2._compat import imap, string_types, text_type, iteritems
 
 
 _word_re = re.compile(r'\w+', re.UNICODE)
diff --git a/jinja2/nativetypes.py b/jinja2/nativetypes.py
index fe17e41..eab95d6 100644
--- a/jinja2/nativetypes.py
+++ b/jinja2/nativetypes.py
@@ -1,11 +1,14 @@
 import sys
 from ast import literal_eval
+
+from markupsafe import escape
+
 from itertools import islice, chain
 from jinja2 import nodes
 from jinja2._compat import text_type
 from jinja2.compiler import CodeGenerator, has_safe_repr
 from jinja2.environment import Environment, Template
-from jinja2.utils import concat, escape
+from jinja2.utils import concat
 
 
 def native_concat(nodes):
diff --git a/jinja2/nodes.py b/jinja2/nodes.py
index 4d9a01a..801764a 100644
--- a/jinja2/nodes.py
+++ b/jinja2/nodes.py
@@ -15,8 +15,9 @@
 import types
 import operator
 
+from markupsafe import Markup
+
 from collections import deque
-from jinja2.utils import Markup
 from jinja2._compat import izip, with_metaclass, text_type, PY2
 
 
diff --git a/jinja2/runtime.py b/jinja2/runtime.py
index 5e31336..c925b71 100644
--- a/jinja2/runtime.py
+++ b/jinja2/runtime.py
@@ -13,8 +13,10 @@ import sys
 from itertools import chain
 from types import MethodType
 
+from markupsafe import Markup, escape, soft_unicode
+
 from jinja2.nodes import EvalContext, _context_function_types
-from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \
+from jinja2.utils import missing, concat, \
      internalcode, object_type_repr, evalcontextfunction, Namespace
 from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \
      TemplateNotFound
diff --git a/jinja2/sandbox.py b/jinja2/sandbox.py
index 08c22f4..88f83ae 100644
--- a/jinja2/sandbox.py
+++ b/jinja2/sandbox.py
@@ -14,13 +14,13 @@
 """
 import types
 import operator
+from string import Formatter
+
+from markupsafe import Markup, EscapeFormatter
+
 from jinja2.environment import Environment
 from jinja2.exceptions import SecurityError
 from jinja2._compat import string_types, PY2, abc, range_type
-from jinja2.utils import Markup
-
-from markupsafe import EscapeFormatter
-from string import Formatter
 
 
 #: maximum number of items a range may produce
diff --git a/jinja2/utils.py b/jinja2/utils.py
index db9c5d0..8c5978d 100644
--- a/jinja2/utils.py
+++ b/jinja2/utils.py
@@ -13,6 +13,9 @@ import json
 import errno
 from collections import deque
 from threading import Lock
+
+from markupsafe import Markup, escape
+
 from jinja2._compat import text_type, string_types, implements_iterator, \
      url_quote, abc
 
@@ -483,25 +486,28 @@ class LRUCache(object):
 abc.MutableMapping.register(LRUCache)
 
 
-def select_autoescape(enabled_extensions=('html', 'htm', 'xml'),
-                      disabled_extensions=(),
-                      default_for_string=True,
-                      default=False):
-    """Intelligently sets the initial value of autoescaping based on the
-    filename of the template.  This is the recommended way to configure
+def select_autoescape(
+    enabled_extensions=("html", "htm", "xml"),
+    disabled_extensions=(),
+    default_for_string=True,
+    default=False,
+):
+    """Set the initial value of autoescaping based on the name of the
+    template, case insensitive. This is the recommended way to configure
     autoescaping if you do not want to write a custom function yourself.
 
-    If you want to enable it for all templates created from strings or
-    for all templates with `.html` and `.xml` extensions::
+    The defaults will enable autoescaping only for template names ending
+    with ".html", ".htm", and ".xml", as well as templates from strings.
+
+    .. code-block:: python
 
         from jinja2 import Environment, select_autoescape
-        env = Environment(autoescape=select_autoescape(
-            enabled_extensions=('html', 'xml'),
-            default_for_string=True,
-        ))
+        env = Environment(autoescape=select_autoescape())
 
-    Example configuration to turn it on at all times except if the template
-    ends with `.txt`::
+    The following configuration enables it for all templates except if
+    the name ends with ".txt".
+
+    .. code-block:: python
 
         from jinja2 import Environment, select_autoescape
         env = Environment(autoescape=select_autoescape(
@@ -510,30 +516,36 @@ def select_autoescape(enabled_extensions=('html', 'htm', 'xml'),
             default=True,
         ))
 
-    The `enabled_extensions` is an iterable of all the extensions that
-    autoescaping should be enabled for.  Likewise `disabled_extensions` is
-    a list of all templates it should be disabled for.  If a template is
-    loaded from a string then the default from `default_for_string` is used.
-    If nothing matches then the initial value of autoescaping is set to the
-    value of `default`.
-
-    For security reasons this function operates case insensitive.
+    :param enabled_extensions: Template names ending in these extensions
+        will have autoescaping enabled. A "." is prepended to each value
+        if it's missing.
+    :param disabled_extensions: Template names ending in these
+        extensions will have autoescaping disabled. A "." is prepended
+        to each value if it's missing.
+    :param default_for_string: What to do if the template is loaded from
+        a string and doesn't have a name.
+    :param default: What to do if the name does not match any of the
+        other rules.
 
     .. versionadded:: 2.9
     """
-    enabled_patterns = tuple('.' + x.lstrip('.').lower()
-                             for x in enabled_extensions)
-    disabled_patterns = tuple('.' + x.lstrip('.').lower()
-                              for x in disabled_extensions)
+    enabled_patterns = tuple("." + x.lstrip(".").lower() for x in enabled_extensions)
+    disabled_patterns = tuple("." + x.lstrip(".").lower() for x in disabled_extensions)
+
     def autoescape(template_name):
         if template_name is None:
             return default_for_string
+
         template_name = template_name.lower()
+
         if template_name.endswith(enabled_patterns):
             return True
+
         if template_name.endswith(disabled_patterns):
             return False
+
         return default
+
     return autoescape
 
 
@@ -636,7 +648,3 @@ try:
     have_async_gen = True
 except SyntaxError:
     have_async_gen = False
-
-
-# Imported here because that's where it was in the past
-from markupsafe import Markup, escape, soft_unicode
diff --git a/tests/test_asyncfilters.py b/tests/test_asyncfilters.py
index 162cc6d..9796caa 100644
--- a/tests/test_asyncfilters.py
+++ b/tests/test_asyncfilters.py
@@ -1,6 +1,8 @@
 import pytest
+
+from markupsafe import Markup
+
 from jinja2 import Environment
-from jinja2.utils import Markup
 
 
 async def make_aiter(iter):
diff --git a/tests/test_filters.py b/tests/test_filters.py
index 8962ced..2769e9c 100644
--- a/tests/test_filters.py
+++ b/tests/test_filters.py
@@ -10,7 +10,10 @@
 """
 import random
 import pytest
-from jinja2 import Markup, Environment
+
+from markupsafe import Markup
+
+from jinja2 import Environment
 from jinja2._compat import text_type, implements_to_string
 
 
diff --git a/tests/test_security.py b/tests/test_security.py
index 5c8639c..6e6b2f2 100644
--- a/tests/test_security.py
+++ b/tests/test_security.py
@@ -10,10 +10,12 @@
 """
 import pytest
 
+from markupsafe import Markup
+
 from jinja2 import Environment
 from jinja2.sandbox import SandboxedEnvironment, \
      ImmutableSandboxedEnvironment, unsafe
-from jinja2 import Markup, escape
+from jinja2 import escape
 from jinja2.exceptions import SecurityError, TemplateSyntaxError, \
      TemplateRuntimeError
 from jinja2.nodes import EvalContext
diff --git a/tests/test_tests.py b/tests/test_tests.py
index 84df5ea..1198516 100644
--- a/tests/test_tests.py
+++ b/tests/test_tests.py
@@ -10,7 +10,9 @@
 """
 import pytest
 
-from jinja2 import Markup, Environment
+from markupsafe import Markup
+
+from jinja2 import Environment
 
 
 @pytest.mark.test_tests