diff options
13 files changed, 364 insertions, 20 deletions
diff --git a/lib/ansible/config/base.yml b/lib/ansible/config/base.yml index deaca279e2..df91d133a0 100644 --- a/lib/ansible/config/base.yml +++ b/lib/ansible/config/base.yml @@ -327,7 +327,7 @@ LOCALHOST_WARNING: description: - By default Ansible will issue a warning when there are no hosts in the inventory. - - These warnings can be silenced by adjusting this setting to False. + - These warnings can be silenced by adjusting this setting to False. env: [{name: ANSIBLE_LOCALHOST_WARNING}] ini: - {key: localhost_warning, section: defaults} @@ -508,7 +508,7 @@ DEFAULT_DEBUG: description: - "Toggles debug output in Ansible. This is *very* verbose and can hinder multiprocessing. Debug output can also include secret information - despite no_log settings being enabled, which means debug mode should not be used in + despite no_log settings being enabled, which means debug mode should not be used in production." env: [{name: ANSIBLE_DEBUG}] ini: @@ -694,6 +694,16 @@ DEFAULT_JINJA2_EXTENSIONS: env: [{name: ANSIBLE_JINJA2_EXTENSIONS}] ini: - {key: jinja2_extensions, section: defaults} +DEFAULT_JINJA2_NATIVE: + name: Use Jinja2's NativeEnvironment for templating + default: False + description: This option preserves variable types during template operations. This requires Jinja2 >= 2.10. + env: [{name: ANSIBLE_JINJA2_NATIVE}] + ini: + - {key: jinja2_native, section: defaults} + type: boolean + yaml: {key: jinja2_native} + version_added: 2.7 DEFAULT_KEEP_REMOTE_FILES: name: Keep remote files default: False diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 401d5e7bb4..9f03a5a40d 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -37,11 +37,9 @@ try: except ImportError: from sha import sha as sha1 -from jinja2 import Environment from jinja2.exceptions import TemplateSyntaxError, UndefinedError from jinja2.loaders import FileSystemLoader from jinja2.runtime import Context, StrictUndefined -from jinja2.utils import concat as j2_concat from ansible import constants as C from ansible.errors import AnsibleError, AnsibleFilterError, AnsibleUndefinedVariable, AnsibleAssertionError @@ -70,6 +68,19 @@ NON_TEMPLATED_TYPES = (bool, Number) JINJA2_OVERRIDE = '#jinja2:' +USE_JINJA2_NATIVE = False +if C.DEFAULT_JINJA2_NATIVE: + try: + from jinja2.nativetypes import NativeEnvironment as Environment + from ansible.template.native_helpers import ansible_native_concat as j2_concat + USE_JINJA2_NATIVE = True + except ImportError: + from jinja2 import Environment + from jinja2.utils import concat as j2_concat +else: + from jinja2 import Environment + from jinja2.utils import concat as j2_concat + def generate_ansible_template_vars(path): b_path = to_bytes(path) @@ -479,19 +490,20 @@ class Templar: disable_lookups=disable_lookups, ) - unsafe = hasattr(result, '__UNSAFE__') - if convert_data and not self._no_type_regex.match(variable): - # if this looks like a dictionary or list, convert it to such using the safe_eval method - if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ - result.startswith("[") or result in ("True", "False"): - eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True) - if eval_results[1] is None: - result = eval_results[0] - if unsafe: - result = wrap_var(result) - else: - # FIXME: if the safe_eval raised an error, should we do something with it? - pass + if not USE_JINJA2_NATIVE: + unsafe = hasattr(result, '__UNSAFE__') + if convert_data and not self._no_type_regex.match(variable): + # if this looks like a dictionary or list, convert it to such using the safe_eval method + if (result.startswith("{") and not result.startswith(self.environment.variable_start_string)) or \ + result.startswith("[") or result in ("True", "False"): + eval_results = safe_eval(result, locals=self._available_variables, include_exceptions=True) + if eval_results[1] is None: + result = eval_results[0] + if unsafe: + result = wrap_var(result) + else: + # FIXME: if the safe_eval raised an error, should we do something with it? + pass # we only cache in the case where we have a single variable # name, to make sure we're not putting things which may otherwise @@ -663,9 +675,15 @@ class Templar: raise AnsibleError("lookup plugin (%s) not found" % name) def do_template(self, data, preserve_trailing_newlines=True, escape_backslashes=True, fail_on_undefined=None, overrides=None, disable_lookups=False): + if USE_JINJA2_NATIVE and not isinstance(data, string_types): + return data + # For preserving the number of input newlines in the output (used # later in this method) - data_newlines = _count_newlines_from_end(data) + if not USE_JINJA2_NATIVE: + data_newlines = _count_newlines_from_end(data) + else: + data_newlines = None if fail_on_undefined is None: fail_on_undefined = self._fail_on_undefined_errors @@ -678,7 +696,7 @@ class Templar: myenv = self.environment.overlay(overrides) # Get jinja env overrides from template - if data.startswith(JINJA2_OVERRIDE): + if hasattr(data, 'startswith') and data.startswith(JINJA2_OVERRIDE): eol = data.find('\n') line = data[len(JINJA2_OVERRIDE):eol] data = data[eol + 1:] @@ -720,7 +738,7 @@ class Templar: try: res = j2_concat(rf) - if new_context.unsafe: + if getattr(new_context, 'unsafe', False): res = wrap_var(res) except TypeError as te: if 'StrictUndefined' in to_native(te): @@ -731,6 +749,9 @@ class Templar: display.debug("failing because of a type error, template data is: %s" % to_native(data)) raise AnsibleError("Unexpected templating type error occurred on (%s): %s" % (to_native(data), to_native(te))) + if USE_JINJA2_NATIVE: + return res + if preserve_trailing_newlines: # The low level calls above do not preserve the newline # characters at the end of the input data, so we use the diff --git a/lib/ansible/template/native_helpers.py b/lib/ansible/template/native_helpers.py new file mode 100644 index 0000000000..d68f849ee7 --- /dev/null +++ b/lib/ansible/template/native_helpers.py @@ -0,0 +1,44 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + + +from ast import literal_eval +from itertools import islice, chain +import types + +from jinja2._compat import text_type + + +def ansible_native_concat(nodes): + """Return a native Python type from the list of compiled nodes. If the + result is a single node, its value is returned. Otherwise, the nodes are + concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the + string is returned. + """ + + # https://github.com/pallets/jinja/blob/master/jinja2/nativetypes.py + + head = list(islice(nodes, 2)) + + if not head: + return None + + if len(head) == 1: + out = head[0] + # short circuit literal_eval when possible + if not isinstance(out, list): # FIXME is this needed? + return out + else: + if isinstance(nodes, types.GeneratorType): + nodes = chain(head, nodes) + out = u''.join([text_type(v) for v in nodes]) + + try: + return literal_eval(out) + except (ValueError, SyntaxError, MemoryError): + return out diff --git a/test/integration/targets/jinja2_native_types/aliases b/test/integration/targets/jinja2_native_types/aliases new file mode 100644 index 0000000000..79d8b9285e --- /dev/null +++ b/test/integration/targets/jinja2_native_types/aliases @@ -0,0 +1 @@ +posix/ci/group3 diff --git a/test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py b/test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py new file mode 100644 index 0000000000..24c716c422 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/filter_plugins/native_plugins.py @@ -0,0 +1,8 @@ +from ansible.module_utils._text import to_text + + +class FilterModule(object): + def filters(self): + return { + 'to_text': to_text, + } diff --git a/test/integration/targets/jinja2_native_types/inventory.jinja2_native_types b/test/integration/targets/jinja2_native_types/inventory.jinja2_native_types new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/inventory.jinja2_native_types diff --git a/test/integration/targets/jinja2_native_types/runme.sh b/test/integration/targets/jinja2_native_types/runme.sh new file mode 100755 index 0000000000..84d7c299f1 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/runme.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eux + +ANSIBLE_JINJA2_NATIVE=1 ansible-playbook -i inventory.jinja2_native_types runtests.yml -v "$@" diff --git a/test/integration/targets/jinja2_native_types/runtests.yml b/test/integration/targets/jinja2_native_types/runtests.yml new file mode 100644 index 0000000000..8d00f471f5 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/runtests.yml @@ -0,0 +1,47 @@ +- name: Test jinja2 native types + hosts: localhost + gather_facts: no + vars: + i_one: 1 + i_two: 2 + i_three: 3 + s_one: "1" + s_two: "2" + s_three: "3" + dict_one: + foo: bar + baz: bang + dict_two: + bar: foo + foobar: barfoo + list_one: + - one + - two + list_two: + - three + - four + list_ints: + - 4 + - 2 + list_one_int: + - 1 + b_true: True + b_false: False + s_true: "True" + s_false: "False" + tasks: + - name: check jinja version + shell: python -c 'import jinja2; print(jinja2.__version__)' + register: jinja2_version + + - name: make sure jinja is the right version + set_fact: + is_native: "{{ jinja2_version.stdout is version('2.10', '>=') }}" + + - block: + - import_tasks: test_casting.yml + - import_tasks: test_concatentation.yml + - import_tasks: test_bool.yml + - import_tasks: test_dunder.yml + - import_tasks: test_types.yml + when: is_native diff --git a/test/integration/targets/jinja2_native_types/test_bool.yml b/test/integration/targets/jinja2_native_types/test_bool.yml new file mode 100644 index 0000000000..f3b5e8c0df --- /dev/null +++ b/test/integration/targets/jinja2_native_types/test_bool.yml @@ -0,0 +1,53 @@ +- name: test bool True + set_fact: + bool_var_true: "{{ b_true }}" + +- assert: + that: + - 'bool_var_true is sameas true' + - 'bool_var_true|type_debug == "bool"' + +- name: test bool False + set_fact: + bool_var_false: "{{ b_false }}" + +- assert: + that: + - 'bool_var_false is sameas false' + - 'bool_var_false|type_debug == "bool"' + +- name: test bool expr True + set_fact: + bool_var_expr_true: "{{ 1 == 1 }}" + +- assert: + that: + - 'bool_var_expr_true is sameas true' + - 'bool_var_expr_true|type_debug == "bool"' + +- name: test bool expr False + set_fact: + bool_var_expr_false: "{{ 2 + 2 == 5 }}" + +- assert: + that: + - 'bool_var_expr_false is sameas false' + - 'bool_var_expr_false|type_debug == "bool"' + +- name: test bool expr with None, True + set_fact: + bool_var_none_expr_true: "{{ None == None }}" + +- assert: + that: + - 'bool_var_none_expr_true is sameas true' + - 'bool_var_none_expr_true|type_debug == "bool"' + +- name: test bool expr with None, False + set_fact: + bool_var_none_expr_false: "{{ '' == None }}" + +- assert: + that: + - 'bool_var_none_expr_false is sameas false' + - 'bool_var_none_expr_false|type_debug == "bool"' diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml new file mode 100644 index 0000000000..7d4e3edaa3 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/test_casting.yml @@ -0,0 +1,24 @@ +- name: cast things to other things + set_fact: + int_to_str: "{{ i_two|to_text }}" + str_to_int: "{{ s_two|int }}" + dict_to_str: "{{ dict_one|to_text }}" + list_to_str: "{{ list_one|to_text }}" + int_to_bool: "{{ i_one|bool }}" + str_true_to_bool: "{{ s_true|bool }}" + str_false_to_bool: "{{ s_false|bool }}" + +- assert: + that: + - 'int_to_str == "2"' + - 'int_to_str|type_debug in ["string", "unicode"]' + - 'str_to_int == 2' + - 'str_to_int|type_debug == "int"' + - 'dict_to_str|type_debug in ["string", "unicode"]' + - 'list_to_str|type_debug in ["string", "unicode"]' + - 'int_to_bool is sameas true' + - 'int_to_bool|type_debug == "bool"' + - 'str_true_to_bool is sameas true' + - 'str_true_to_bool|type_debug == "bool"' + - 'str_false_to_bool is sameas false' + - 'str_false_to_bool|type_debug == "bool"' diff --git a/test/integration/targets/jinja2_native_types/test_concatentation.yml b/test/integration/targets/jinja2_native_types/test_concatentation.yml new file mode 100644 index 0000000000..9a523e543d --- /dev/null +++ b/test/integration/targets/jinja2_native_types/test_concatentation.yml @@ -0,0 +1,88 @@ +- name: add two ints + set_fact: + integer_sum: "{{ i_one + i_two }}" + +- assert: + that: + - 'integer_sum == 3' + - 'integer_sum|type_debug == "int"' + +- name: add casted string and int + set_fact: + integer_sum2: "{{ s_one|int + i_two }}" + +- assert: + that: + - 'integer_sum2 == 3' + - 'integer_sum2|type_debug == "int"' + +- name: concatenate int and string + set_fact: + string_sum: "{{ [(i_one|to_text), s_two]|join('') }}" + +- assert: + that: + - 'string_sum == "12"' + - 'string_sum|type_debug in ["string", "unicode"]' + +- name: add two lists + set_fact: + list_sum: "{{ list_one + list_two }}" + +- assert: + that: + - 'list_sum == ["one", "two", "three", "four"]' + - 'list_sum|type_debug == "list"' + +- name: add two lists, multi expression + set_fact: + list_sum_multi: "{{ list_one }} + {{ list_two }}" + +- assert: + that: + - 'list_sum_multi|type_debug in ["string", "unicode"]' + +- name: add two dicts + set_fact: + dict_sum: "{{ dict_one + dict_two }}" + ignore_errors: yes + +- assert: + that: + - 'dict_sum is undefined' + +- name: loop through list with strings + set_fact: + list_for_strings: "{% for x in list_one %}{{ x }}{% endfor %}" + +- assert: + that: + - 'list_for_strings == "onetwo"' + - 'list_for_strings|type_debug in ["string", "unicode"]' + +- name: loop through list with int + set_fact: + list_for_int: "{% for x in list_one_int %}{{ x }}{% endfor %}" + +- assert: + that: + - 'list_for_int == 1' + - 'list_for_int|type_debug == "int"' + +- name: loop through list with ints + set_fact: + list_for_ints: "{% for x in list_ints %}{{ x }}{% endfor %}" + +- assert: + that: + - 'list_for_ints == 42' + - 'list_for_ints|type_debug == "int"' + +- name: loop through list to create a new list + set_fact: + list_from_list: "[{% for x in list_ints %}{{ x }},{% endfor %}]" + +- assert: + that: + - 'list_from_list == [4, 2]' + - 'list_from_list|type_debug == "list"' diff --git a/test/integration/targets/jinja2_native_types/test_dunder.yml b/test/integration/targets/jinja2_native_types/test_dunder.yml new file mode 100644 index 0000000000..798e771027 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/test_dunder.yml @@ -0,0 +1,23 @@ +- name: test variable dunder + set_fact: + var_dunder: "{{ b_true.__class__ }}" + +- assert: + that: + - 'var_dunder|type_debug == "type"' + +- name: test constant dunder + set_fact: + const_dunder: "{{ true.__class__ }}" + +- assert: + that: + - 'const_dunder|type_debug == "type"' + +- name: test constant dunder to string + set_fact: + const_dunder: "{{ true.__class__|string }}" + +- assert: + that: + - 'const_dunder|type_debug in ["string", "unicode"]' diff --git a/test/integration/targets/jinja2_native_types/test_types.yml b/test/integration/targets/jinja2_native_types/test_types.yml new file mode 100644 index 0000000000..f5659d4e15 --- /dev/null +++ b/test/integration/targets/jinja2_native_types/test_types.yml @@ -0,0 +1,20 @@ +- assert: + that: + - 'i_one|type_debug == "int"' + - 's_one|type_debug == "AnsibleUnicode"' + - 'dict_one|type_debug == "dict"' + - 'dict_one is mapping' + - 'list_one|type_debug == "list"' + - 'b_true|type_debug == "bool"' + - 's_true|type_debug == "AnsibleUnicode"' + +- set_fact: + a_list: "{{[i_one, s_two]}}" + +- assert: + that: + - 'a_list|type_debug == "list"' + - 'a_list[0] == 1' + - 'a_list[0]|type_debug == "int"' + - 'a_list[1] == "2"' + - 'a_list[1]|type_debug == "AnsibleUnicode"' |