fix decrypted vault utf8 values (#37539)

* Fix errors decrypted non-ascii vault vars

AnsibleVaultEncryptedUnicode was just using b"".decode()
instead of to_text() on the bytestrings returned from
vault.decrypt() and could cause errors on python2
if non-ascii since decode() defaults to ascii.
Use to_text() to default to decoding utf-8.

add intg and unit tests for value of vaulted vars
being non-ascii utf8

based on https://github.com/ansible/ansible/issues/37258

Fixes #37258

* yamllint fixups

6 files changed, 40 insertions, 2 deletions

diff --git a/lib/ansible/parsing/yaml/objects.py b/lib/ansible/parsing/yaml/objects.py
index 45cbb4d5dd..c161692b33 100644
--- a/lib/ansible/parsing/yaml/objects.py
+++ b/lib/ansible/parsing/yaml/objects.py
@@ -104,7 +104,7 @@ class AnsibleVaultEncryptedUnicode(yaml.YAMLObject, AnsibleBaseYAMLObject):
         if not self.vault:
             # FIXME: raise exception?
             return self._ciphertext
-        return self.vault.decrypt(self._ciphertext).decode()
+        return to_text(self.vault.decrypt(self._ciphertext))
 
     @data.setter
     def data(self, value):
diff --git a/test/integration/targets/vault/host_vars/myhost.yml b/test/integration/targets/vault/host_vars/myhost.yml
new file mode 100644
index 0000000000..1434ec15eb
--- /dev/null
+++ b/test/integration/targets/vault/host_vars/myhost.yml
@@ -0,0 +1,7 @@
+myvar: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          31356335363836383937363933366135623233343830326234633633623734336636343630396464
+          3234343638313166663237343536646336323862613739380a346266316336356230643838663031
+          34623034383639323062373235356564393337346666393665313237313231306131356637346537
+          3966393238666430310a363462326639323033653237373036643936613234623063643761663033
+          3832
diff --git a/test/integration/targets/vault/host_vars/testhost.yml b/test/integration/targets/vault/host_vars/testhost.yml
new file mode 100644
index 0000000000..b3e569ad66
--- /dev/null
+++ b/test/integration/targets/vault/host_vars/testhost.yml
@@ -0,0 +1,7 @@
+vaulted_utf8_value: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          39313961356631343234656136636231663539363963386364653436346133366366633031366364
+          3332376636333837333036633662316135383365343335380a393331663434663238666537343163
+          62363561336431623666633735313766613663333736653064373632666131356434336537383336
+          3333343436613232330a643461363831633166333237653530353131316361643465353132616362
+          3461
diff --git a/test/integration/targets/vault/runme.sh b/test/integration/targets/vault/runme.sh
index 5b9748926e..0de80d63bc 100755
--- a/test/integration/targets/vault/runme.sh
+++ b/test/integration/targets/vault/runme.sh
@@ -386,7 +386,7 @@ echo "rc was $WRONG_RC (5 is expected)"
 
 # encrypt with a password from a vault encrypted password file and multiple vault-ids
 # but this time specify with --encrypt-vault-id, but specifying vault-id names (instead of default)
-# ansible-vault encrypt "$@" --vault-id from_vault_password@vault-password --vault-id from_encrypted_vault_password@encrypted-vault-password --encrypt-vault-id from_encrypted_vault_password "${TEST_FILE_ENC_PASSWORD}"
+# ansible-vault encrypt "$@" --vault-id from_vault_password@vault-password --vault-id from_encrypted_vault_password@encrypted-vault-password --encrypt-vault-id from_encrypted_vault_password "${TEST_FILE(_ENC_PASSWORD}"
 
 # try to view the file encrypted with the vault-password we didnt specify
 # to verify we didnt choose the wrong vault-id
@@ -418,6 +418,9 @@ ansible-playbook test_vault_embedded.yml -i ../../inventory -v "$@" --vault-pass
 ansible-playbook test_vaulted_inventory.yml -i vaulted.inventory -v "$@" --vault-password-file vault-password
 ansible-playbook test_vaulted_template.yml -i ../../inventory -v "$@" --vault-password-file vault-password
 
+# test a playbook with a host_var whose value is non-ascii utf8 (see https://github.com/ansible/ansible/issues/37258)
+ansible-playbook -i ../../inventory -v "$@" --vault-id vault-password test_vaulted_utf8_value.yml
+
 # test with password from password script
 ansible-playbook test_vault.yml          -i ../../inventory -v "$@" --vault-password-file password-script.py
 ansible-playbook test_vault_embedded.yml -i ../../inventory -v "$@" --vault-password-file password-script.py
diff --git a/test/integration/targets/vault/test_vaulted_utf8_value.yml b/test/integration/targets/vault/test_vaulted_utf8_value.yml
new file mode 100644
index 0000000000..63b602b15a
--- /dev/null
+++ b/test/integration/targets/vault/test_vaulted_utf8_value.yml
@@ -0,0 +1,15 @@
+- name: "test that the vaulted_utf8_value decrypts correctly"
+  gather_facts: false
+  hosts: testhost
+  vars:
+    expected: "aöffü"
+  tasks:
+    - name: decrypt vaulted_utf8_value and show it in debug
+      debug:
+         var: vaulted_utf8_value
+
+    - name: assert decrypted vaulted_utf8_value matches expected
+      assert:
+        that:
+          - "vaulted_utf8_value == expected"
+          - "vaulted_utf8_value == 'aöffü'"
diff --git a/test/units/parsing/yaml/test_objects.py b/test/units/parsing/yaml/test_objects.py
index 71fa7670b8..6be7fbb783 100644
--- a/test/units/parsing/yaml/test_objects.py
+++ b/test/units/parsing/yaml/test_objects.py
@@ -1,4 +1,5 @@
 # This file is part of Ansible
+# -*- coding: utf-8 -*-
 #
 # Ansible is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -149,3 +150,8 @@ class TestAnsibleVaultEncryptedUnicode(unittest.TestCase, YamlTestUtils):
             return avu == seq
 
         self.assertRaises(AnsibleError, compare, avu, seq)
+
+    def test_vaulted_utf8_value_37258(self):
+        seq = u"aöffü"
+        avu = self._from_plaintext(seq)
+        self.assert_values(avu, seq)