Overhaul ansible-test test path handling. (#61416)

* Remove .keep files from test/results/ dirs.

* Remove classification of test/results/ dir.

* Add results_relative to data context.

* Use variables in delegation paths.

* Standardize file writing and results paths.

* Fix issues reported by PyCharm.

* Clean up invocation of coverage command.

It now runs through the injector.

* Hack to allow intercept_command in cover.py.

* Simplify git ignore for test results.

* Use test result tmp dir instead of cache dir.

* Remove old .pytest_cache reference.

* Fix unit test docker delegation.

* Show HTML report link.

* Clean up more results references.

* Move import sanity test output to .tmp dir.

* Exclude test results dir from coverage.

* Fix import sanity test lib paths.

* Fix hard-coded import test paths.

* Fix most hard-coded integration test paths.

* Fix PyCharm warnings.

* Fix import placement.

* Fix integration test dir path.

* Fix Shippable scripts.

* Fix Shippable matrix check.

* Overhaul key pair management.

2 files changed, 61 insertions, 46 deletions

diff --git a/test/utils/shippable/check_matrix.py b/test/utils/shippable/check_matrix.py
index 99e4ea88ff..c522f3ab98 100755
--- a/test/utils/shippable/check_matrix.py
+++ b/test/utils/shippable/check_matrix.py
@@ -94,7 +94,13 @@ def fail(message, output):  # type: (str, str) -> NoReturn
 </testsuites>
 ''' % (timestamp, message, output)
 
-    with open('test/results/junit/check-matrix.xml', 'w') as junit_fd:
+    path = 'shippable/testresults/check-matrix.xml'
+    dir_path = os.path.dirname(path)
+
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+
+    with open(path, 'w') as junit_fd:
         junit_fd.write(xml.lstrip())
 
     sys.stderr.write(message + '\n')
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index e47b85eafc..a5bee2761e 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -73,55 +73,64 @@ find lib/ansible/modules -type d -empty -print -delete
 
 function cleanup
 {
-    if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-        # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
-        if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then
-            stub="--stub"
-        else
-            stub=""
+    if [ -d test/results/coverage/ ]; then
+        if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
+            # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then
+                stub="--stub"
+            else
+                stub=""
+            fi
+
+            # use python 3.7 for coverage to avoid running out of memory during coverage xml processing
+            # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job
+            virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+            set +ux
+            . ~/ansible-venv/bin/activate
+            set -ux
+
+            # shellcheck disable=SC2086
+            ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
+            cp -a test/results/reports/coverage=*.xml shippable/codecoverage/
+
+            # upload coverage report to codecov.io only when using complete on-demand coverage
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
+                for file in test/results/reports/coverage=*.xml; do
+                    flags="${file##*/coverage=}"
+                    flags="${flags%-powershell.xml}"
+                    flags="${flags%.xml}"
+                    # remove numbered component from stub files when converting to tags
+                    flags="${flags//stub-[0-9]*/stub}"
+                    flags="${flags//=/,}"
+                    flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                    bash <(curl -s https://codecov.io/bash) \
+                        -f "${file}" \
+                        -F "${flags}" \
+                        -n "${test}" \
+                        -t 83cd8957-dc76-488c-9ada-210dcea51633 \
+                        -X coveragepy \
+                        -X gcov \
+                        -X fix \
+                        -X search \
+                        -X xcode \
+                    || echo "Failed to upload code coverage report to codecov.io: ${file}"
+                done
+            fi
         fi
+    fi
 
-        # use python 3.7 for coverage to avoid running out of memory during coverage xml processing
-        # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job
-        virtualenv --python /usr/bin/python3.7 ~/ansible-venv
-        set +ux
-        . ~/ansible-venv/bin/activate
-        set -ux
-
-        # shellcheck disable=SC2086
-        ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
-        cp -a test/results/reports/coverage=*.xml shippable/codecoverage/
-
-        # upload coverage report to codecov.io only when using complete on-demand coverage
-        if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
-            for file in test/results/reports/coverage=*.xml; do
-                flags="${file##*/coverage=}"
-                flags="${flags%-powershell.xml}"
-                flags="${flags%.xml}"
-                # remove numbered component from stub files when converting to tags
-                flags="${flags//stub-[0-9]*/stub}"
-                flags="${flags//=/,}"
-                flags="${flags//[^a-zA-Z0-9_,]/_}"
-
-                bash <(curl -s https://codecov.io/bash) \
-                    -f "${file}" \
-                    -F "${flags}" \
-                    -n "${test}" \
-                    -t 83cd8957-dc76-488c-9ada-210dcea51633 \
-                    -X coveragepy \
-                    -X gcov \
-                    -X fix \
-                    -X search \
-                    -X xcode \
-                || echo "Failed to upload code coverage report to codecov.io: ${file}"
-            done
-        fi
+    if [ -d test/results/junit/ ]; then
+      cp -a test/results/junit/ shippable/testresults/
     fi
 
-    rmdir shippable/testresults/
-    cp -a test/results/junit/ shippable/testresults/
-    cp -a test/results/data/ shippable/testresults/
-    cp -aT test/results/bot/ shippable/testresults/
+    if [ -d test/results/data/ ]; then
+      cp -a test/results/data/ shippable/testresults/
+    fi
+
+    if [ -d  test/results/bot/ ]; then
+      cp -aT test/results/bot/ shippable/testresults/
+    fi
 }
 
 trap cleanup EXIT