[stable-2.9] Fix ansible-test PYTHONPATH handling.

Running from an installed version of ansible-test now results in tests using a dedicated directory for PYTHONPATH instead of using the site-packages directory where ansible is installed.

This provides consistency with tests running from source, which already used a dedicated directory.

Resolves https://github.com/ansible/ansible/issues/62716
(cherry picked from commit 831e1bf2e0375446c4a4ee04c1831b1b9b683e94)

Co-authored-by: Matt Clay <matt@mystile.com>

3 files changed, 35 insertions, 1 deletions

diff --git a/changelogs/fragments/ansible-test-fix-python-path.yml b/changelogs/fragments/ansible-test-fix-python-path.yml
new file mode 100644
index 0000000000..9de7dfd887
--- /dev/null
+++ b/changelogs/fragments/ansible-test-fix-python-path.yml
@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now properly sets PYTHONPATH for tests when running from an Ansible installation
diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py
index e5b4a46ef9..f26b983594 100644
--- a/test/lib/ansible_test/_internal/ansible_util.py
+++ b/test/lib/ansible_test/_internal/ansible_util.py
@@ -17,9 +17,11 @@ from .util import (
     ANSIBLE_LIB_ROOT,
     ANSIBLE_TEST_DATA_ROOT,
     ANSIBLE_BIN_PATH,
+    ANSIBLE_SOURCE_ROOT,
 )
 
 from .util_common import (
+    create_temp_dir,
     run_command,
     ResultType,
 )
@@ -68,7 +70,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         ANSIBLE_RETRY_FILES_ENABLED='false',
         ANSIBLE_CONFIG=ansible_config,
         ANSIBLE_LIBRARY='/dev/null',
-        PYTHONPATH=os.path.dirname(ANSIBLE_LIB_ROOT),
+        PYTHONPATH=get_ansible_python_path(),
         PAGER='/bin/cat',
         PATH=path,
     )
@@ -94,6 +96,28 @@ def ansible_environment(args, color=True, ansible_config=None):
     return env
 
 
+def get_ansible_python_path():  # type: () -> str
+    """
+    Return a directory usable for PYTHONPATH, containing only the ansible package.
+    If a temporary directory is required, it will be cached for the lifetime of the process and cleaned up at exit.
+    """
+    if ANSIBLE_SOURCE_ROOT:
+        # when running from source there is no need for a temporary directory to isolate the ansible package
+        return os.path.dirname(ANSIBLE_LIB_ROOT)
+
+    try:
+        return get_ansible_python_path.python_path
+    except AttributeError:
+        pass
+
+    python_path = create_temp_dir(prefix='ansible-test-')
+    get_ansible_python_path.python_path = python_path
+
+    os.symlink(ANSIBLE_LIB_ROOT, os.path.join(python_path, 'ansible'))
+
+    return python_path
+
+
 def check_pyyaml(args, version):
     """
     :type args: EnvironmentConfig
diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py
index 5f601fbb67..f4035ee934 100644
--- a/test/lib/ansible_test/_internal/util_common.py
+++ b/test/lib/ansible_test/_internal/util_common.py
@@ -19,6 +19,7 @@ from .util import (
     display,
     find_python,
     is_shippable,
+    remove_tree,
     MODE_DIRECTORY,
     MODE_FILE_EXECUTE,
     PYTHON_PATHS,
@@ -214,6 +215,13 @@ def get_python_path(args, interpreter):
     return python_path
 
 
+def create_temp_dir(prefix=None, suffix=None, base_dir=None):  # type: (t.Optional[str], t.Optional[str], t.Optional[str]) -> str
+    """Create a temporary directory that persists until the current process exits."""
+    temp_path = tempfile.mkdtemp(prefix=prefix or 'tmp', suffix=suffix or '', dir=base_dir)
+    atexit.register(remove_tree, temp_path)
+    return temp_path
+
+
 def create_interpreter_wrapper(interpreter, injected_interpreter):  # type: (str, str) -> None
     """Create a wrapper for the given Python interpreter at the specified path."""
     # sys.executable is used for the shebang to guarantee it is a binary instead of a script