Fix ansible-test submodule handling. (#68759)

* Refactor ansible-test integration test.
* Add env --list-files option.
* Add tests for collection files tracked by git.
* Fix ansible-test submodule usage on older git.
* Fix submodule directory detection as files.
* Improve handling of nested source control.

14 files changed, 149 insertions, 29 deletions

diff --git a/changelogs/fragments/ansible-test-env-list-files.yml b/changelogs/fragments/ansible-test-env-list-files.yml
new file mode 100644
index 0000000000..aae86817a5
--- /dev/null
+++ b/changelogs/fragments/ansible-test-env-list-files.yml
@@ -0,0 +1,2 @@
+minor_changes:
+    - ansible-test now has a ``--list-files`` option to list files using the ``env`` command.
diff --git a/changelogs/fragments/ansible-test-git-submodule-top-level.yml b/changelogs/fragments/ansible-test-git-submodule-top-level.yml
new file mode 100644
index 0000000000..6d29cc6ad5
--- /dev/null
+++ b/changelogs/fragments/ansible-test-git-submodule-top-level.yml
@@ -0,0 +1,2 @@
+bugfixes:
+    - ansible-test now supports submodules using older ``git`` versions which require querying status from the top level directory of the repo.
diff --git a/changelogs/fragments/ansible-test-nested-source-control.yml b/changelogs/fragments/ansible-test-nested-source-control.yml
new file mode 100644
index 0000000000..85a347c53c
--- /dev/null
+++ b/changelogs/fragments/ansible-test-nested-source-control.yml
@@ -0,0 +1,2 @@
+bugfixes:
+  - ansible-test now ignores version control within subdirectories of collections. Previously this condition was an error.
diff --git a/changelogs/fragments/ansible-test-submodule-dir-as-file.yml b/changelogs/fragments/ansible-test-submodule-dir-as-file.yml
new file mode 100644
index 0000000000..fef8b6c067
--- /dev/null
+++ b/changelogs/fragments/ansible-test-submodule-dir-as-file.yml
@@ -0,0 +1,2 @@
+minor_changes:
+    - ansible-test no longer detects ``git`` submodule directories as files.
diff --git a/test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh b/test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh
new file mode 100755
index 0000000000..31ebfbbfc3
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/git-at-collection-base.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+export GIT_TOP_LEVEL SUBMODULE_DST
+
+GIT_TOP_LEVEL="${WORK_DIR}/super/ansible_collections/ns/col"
+SUBMODULE_DST="sub"
+
+source collection-tests/git-common.bash
diff --git a/test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh b/test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh
new file mode 100755
index 0000000000..8af4387a3d
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/git-at-collection-root.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+export GIT_TOP_LEVEL SUBMODULE_DST
+
+GIT_TOP_LEVEL="${WORK_DIR}/super"
+SUBMODULE_DST="ansible_collections/ns/col/sub"
+
+source collection-tests/git-common.bash
diff --git a/test/integration/targets/ansible-test/collection-tests/git-common.bash b/test/integration/targets/ansible-test/collection-tests/git-common.bash
new file mode 100755
index 0000000000..069b157c65
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/git-common.bash
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+# make sure git is installed
+git --version || ansible-playbook collection-tests/install-git.yml -i ../../inventory "$@"
+
+# init sub project
+mkdir "${WORK_DIR}/sub"
+cd "${WORK_DIR}/sub"
+touch "README.md"
+git init
+git config user.name 'Ansible Test'
+git config user.email 'ansible-test@ansible.com'
+git add "README.md"
+git commit -m "Initial commit."
+
+# init super project
+rm -rf "${WORK_DIR}/super" # needed when re-creating in place
+mkdir "${WORK_DIR}/super"
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}/super"
+cd "${GIT_TOP_LEVEL}"
+git init
+
+# add submodule
+git submodule add "${WORK_DIR}/sub" "${SUBMODULE_DST}"
+
+# prepare for tests
+expected="${WORK_DIR}/expected.txt"
+actual="${WORK_DIR}/actual.txt"
+cd "${WORK_DIR}/super/ansible_collections/ns/col"
+mkdir tests/.git
+touch tests/.git/keep.txt  # make sure ansible-test correctly ignores version control within collection subdirectories
+find . -type f ! -path '*/.git/*' ! -name .git | sed 's|^\./||' | sort >"${expected}"
+set -x
+
+# test at the collection base
+ansible-test env --list-files | sort >"${actual}"
+diff --unified "${expected}" "${actual}"
+
+# test at the submodule base
+(cd sub && ansible-test env --list-files | sort >"${actual}")
+diff --unified "${expected}" "${actual}"
diff --git a/test/integration/targets/ansible-test/collection-tests/install-git.yml b/test/integration/targets/ansible-test/collection-tests/install-git.yml
new file mode 100644
index 0000000000..29adead7de
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/install-git.yml
@@ -0,0 +1,5 @@
+- hosts: localhost
+  tasks:
+    - name: Make sure git is installed
+      package:
+        name: git
diff --git a/test/integration/targets/ansible-test/collection-tests/venv.sh b/test/integration/targets/ansible-test/collection-tests/venv.sh
new file mode 100755
index 0000000000..45f41bcd85
--- /dev/null
+++ b/test/integration/targets/ansible-test/collection-tests/venv.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+
+cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
+cd "${WORK_DIR}/ansible_collections/ns/col"
+
+# common args for all tests
+common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
+
+# prime the venv to work around issue with PyYAML detection in ansible-test
+ansible-test sanity "${common[@]}" --test ignores
+
+# tests
+ansible-test sanity "${common[@]}"
+ansible-test units "${common[@]}"
+ansible-test integration "${common[@]}"
diff --git a/test/integration/targets/ansible-test/runme.sh b/test/integration/targets/ansible-test/runme.sh
index d3c40f522d..7c956b4f15 100755
--- a/test/integration/targets/ansible-test/runme.sh
+++ b/test/integration/targets/ansible-test/runme.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eux -o pipefail
+set -eu -o pipefail
 
 # tests must be executed outside of the ansible source tree
 # otherwise ansible-test will test the ansible source instead of the test collection
@@ -9,16 +9,16 @@ tmp_dir=$(mktemp -d)
 
 trap 'rm -rf "${tmp_dir}"' EXIT
 
-cp -a ansible_collections "${tmp_dir}"
-cd "${tmp_dir}/ansible_collections/ns/col"
+export TEST_DIR
+export WORK_DIR
 
-# common args for all tests
-common=(--venv --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --truncate 0 "${@}")
+TEST_DIR="$PWD"
 
-# prime the venv to work around issue with PyYAML detection in ansible-test
-ansible-test sanity "${common[@]}" --test ignores
-
-# tests
-ansible-test sanity "${common[@]}"
-ansible-test units "${common[@]}"
-ansible-test integration "${common[@]}"
+for test in collection-tests/*.sh; do
+  WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")"
+  mkdir "${WORK_DIR}"
+  echo "**********************************************************************"
+  echo "TEST: ${test}: STARTING"
+  "${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1)
+  echo "TEST: ${test}: PASSED"
+done
diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py
index 942ef905cc..0c8345718b 100644
--- a/test/lib/ansible_test/_internal/cli.py
+++ b/test/lib/ansible_test/_internal/cli.py
@@ -636,6 +636,10 @@ def parse_args():
                      action='store_true',
                      help='dump environment to disk')
 
+    env.add_argument('--list-files',
+                     action='store_true',
+                     help='list files on stdout')
+
     # noinspection PyTypeChecker
     env.add_argument('--timeout',
                      type=int,
diff --git a/test/lib/ansible_test/_internal/data.py b/test/lib/ansible_test/_internal/data.py
index 6f8cbf45ca..38ae6d210e 100644
--- a/test/lib/ansible_test/_internal/data.py
+++ b/test/lib/ansible_test/_internal/data.py
@@ -40,15 +40,6 @@ from .provider.layout import (
 )
 
 
-class UnexpectedSourceRoot(ApplicationError):
-    """Exception generated when a source root is found below a layout root."""
-    def __init__(self, source_root, layout_root):  # type: (str, str) -> None
-        super(UnexpectedSourceRoot, self).__init__('Source root "%s" cannot be below layout root "%s".' % (source_root, layout_root))
-
-        self.source_root = source_root
-        self.layout_root = layout_root
-
-
 class DataContext:
     """Data context providing details about the current execution environment for ansible-test."""
     def __init__(self):
@@ -121,13 +112,14 @@ class DataContext:
         layout_provider = find_path_provider(LayoutProvider, layout_providers, root, walk)
 
         try:
-            source_provider = find_path_provider(SourceProvider, source_providers, root, walk)
+            # Begin the search for the source provider at the layout provider root.
+            # This intentionally ignores version control within subdirectories of the layout root, a condition which was previously an error.
+            # Doing so allows support for older git versions for which it is difficult to distinguish between a super project and a sub project.
+            # It also provides a better user experience, since the solution for the user would effectively be the same -- to remove the nested version control.
+            source_provider = find_path_provider(SourceProvider, source_providers, layout_provider.root, walk)
         except ProviderNotFoundForPath:
             source_provider = UnversionedSource(layout_provider.root)
 
-        if source_provider.root != layout_provider.root and is_subdir(source_provider.root, layout_provider.root):
-            raise UnexpectedSourceRoot(source_provider.root, layout_provider.root)
-
         layout = layout_provider.create(layout_provider.root, source_provider.get_paths(layout_provider.root))
 
         return layout
diff --git a/test/lib/ansible_test/_internal/env.py b/test/lib/ansible_test/_internal/env.py
index 31d61a8faf..f495981c16 100644
--- a/test/lib/ansible_test/_internal/env.py
+++ b/test/lib/ansible_test/_internal/env.py
@@ -31,6 +31,7 @@ from .util import (
 )
 
 from .util_common import (
+    data_context,
     write_json_test_results,
     ResultType,
 )
@@ -72,8 +73,9 @@ class EnvConfig(CommonConfig):
         self.show = args.show
         self.dump = args.dump
         self.timeout = args.timeout
+        self.list_files = args.list_files
 
-        if not self.show and not self.dump and self.timeout is None:
+        if not self.show and not self.dump and self.timeout is None and not self.list_files:
             # default to --show if no options were given
             self.show = True
 
@@ -83,6 +85,7 @@ def command_env(args):
     :type args: EnvConfig
     """
     show_dump_env(args)
+    list_files_env(args)
     set_timeout(args)
 
 
@@ -130,6 +133,15 @@ def show_dump_env(args):
         write_json_test_results(ResultType.BOT, 'data-environment.json', data)
 
 
+def list_files_env(args):  # type: (EnvConfig) -> None
+    """List files on stdout."""
+    if not args.list_files:
+        return
+
+    for path in data_context().content.all_files():
+        display.info(path)
+
+
 def set_timeout(args):
     """
     :type args: EnvConfig
diff --git a/test/lib/ansible_test/_internal/provider/source/git.py b/test/lib/ansible_test/_internal/provider/source/git.py
index 081fbc44e2..0bf81a1cee 100644
--- a/test/lib/ansible_test/_internal/provider/source/git.py
+++ b/test/lib/ansible_test/_internal/provider/source/git.py
@@ -14,6 +14,10 @@ from ...encoding import (
     to_bytes,
 )
 
+from ...util import (
+    SubprocessError,
+)
+
 from . import (
     SourceProvider,
 )
@@ -28,15 +32,30 @@ class GitSource(SourceProvider):
 
     def get_paths(self, path):  # type: (str) -> t.List[str]
         """Return the list of available content paths under the given path."""
-        git = Git(path)
-
         paths = self.__get_paths(path)
 
-        submodule_paths = git.get_submodule_paths()
+        try:
+            submodule_paths = Git(path).get_submodule_paths()
+        except SubprocessError:
+            if path == self.root:
+                raise
+
+            # older versions of git require submodule commands to be executed from the top level of the working tree
+            # git version 2.18.1 (centos8) does not have this restriction
+            # git version 1.8.3.1 (centos7) does
+            # fall back to using the top level directory of the working tree only when needed
+            # this avoids penalizing newer git versions with a potentially slower analysis due to additional submodules
+            rel_path = os.path.relpath(path, self.root) + os.path.sep
+
+            submodule_paths = Git(self.root).get_submodule_paths()
+            submodule_paths = [os.path.relpath(p, rel_path) for p in submodule_paths if p.startswith(rel_path)]
 
         for submodule_path in submodule_paths:
             paths.extend(os.path.join(submodule_path, p) for p in self.__get_paths(os.path.join(path, submodule_path)))
 
+        # git reports submodule directories as regular files
+        paths = [p for p in paths if p not in submodule_paths]
+
         return paths
 
     @staticmethod