Fix ansible-test python and pip executable search.

14 files changed, 81 insertions, 75 deletions

diff --git a/test/runner/lib/cloud/tower.py b/test/runner/lib/cloud/tower.py
index 29e147f9e9..ac64d725f7 100644
--- a/test/runner/lib/cloud/tower.py
+++ b/test/runner/lib/cloud/tower.py
@@ -15,7 +15,6 @@ from lib.util import (
     display,
     ApplicationError,
     is_shippable,
-    find_pip,
     run_command,
     generate_password,
     SubprocessError,
@@ -151,8 +150,7 @@ class TowerCloudEnvironment(CloudEnvironment):
 
         display.info('Installing Tower CLI version: %s' % tower_cli_version)
 
-        pip = find_pip(version=self.args.python_version)
-        cmd = [pip, 'install', '--disable-pip-version-check', 'ansible-tower-cli==%s' % tower_cli_version]
+        cmd = self.args.pip_command + ['install', '--disable-pip-version-check', 'ansible-tower-cli==%s' % tower_cli_version]
 
         run_command(self.args, cmd)
 
diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py
index eb6b5f665e..b1f23954fe 100644
--- a/test/runner/lib/config.py
+++ b/test/runner/lib/config.py
@@ -9,6 +9,8 @@ from lib.util import (
     CommonConfig,
     is_shippable,
     docker_qualify_image,
+    find_python,
+    generate_pip_command,
 )
 
 from lib.metadata import (
@@ -67,6 +69,20 @@ class EnvironmentConfig(CommonConfig):
         if self.delegate:
             self.requirements = True
 
+    @property
+    def python_executable(self):
+        """
+        :rtype: str
+        """
+        return find_python(self.python_version)
+
+    @property
+    def pip_command(self):
+        """
+        :rtype: list[str]
+        """
+        return generate_pip_command(self.python_executable)
+
 
 class TestConfig(EnvironmentConfig):
     """Configuration common to all test commands."""
diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py
index 2f2b86973c..119b0f889d 100644
--- a/test/runner/lib/executor.py
+++ b/test/runner/lib/executor.py
@@ -45,7 +45,6 @@ from lib.util import (
     make_dirs,
     is_shippable,
     is_binary_file,
-    find_pip,
     find_executable,
     raw_command,
     get_coverage_path,
@@ -151,7 +150,7 @@ def install_command_requirements(args):
         if args.junit:
             packages.append('junit-xml')
 
-    pip = find_pip(version=args.python_version)
+    pip = args.pip_command
 
     commands = [generate_pip_install(pip, args.command, packages=packages)]
 
@@ -183,7 +182,7 @@ def install_command_requirements(args):
 def run_pip_commands(args, pip, commands, detect_pip_changes=False):
     """
     :type args: EnvironmentConfig
-    :type pip: str
+    :type pip: list[str]
     :type commands: list[list[str]]
     :type detect_pip_changes: bool
     :rtype: list[list[str]]
@@ -210,7 +209,7 @@ def run_pip_commands(args, pip, commands, detect_pip_changes=False):
             # AttributeError: 'Requirement' object has no attribute 'project_name'
             # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
             # Upgrading pip works around the issue.
-            run_command(args, [pip, 'install', '--upgrade', 'pip'])
+            run_command(args, pip + ['install', '--upgrade', 'pip'])
             run_command(args, cmd)
 
         after_list = pip_list(args, pip) if detect_pip_changes else None
@@ -224,10 +223,10 @@ def run_pip_commands(args, pip, commands, detect_pip_changes=False):
 def pip_list(args, pip):
     """
     :type args: EnvironmentConfig
-    :type pip: str
+    :type pip: list[str]
     :rtype: str
     """
-    stdout, _ = run_command(args, [pip, 'list'], capture=True)
+    stdout, _ = run_command(args, pip + ['list'], capture=True)
     return stdout
 
 
@@ -238,12 +237,12 @@ def generate_egg_info(args):
     if os.path.isdir('lib/ansible.egg-info'):
         return
 
-    run_command(args, ['python%s' % args.python_version, 'setup.py', 'egg_info'], capture=args.verbosity < 3)
+    run_command(args, [args.python_executable, 'setup.py', 'egg_info'], capture=args.verbosity < 3)
 
 
 def generate_pip_install(pip, command, packages=None):
     """
-    :type pip: str
+    :type pip: list[str]
     :type command: str
     :type packages: list[str] | None
     :rtype: list[str] | None
@@ -262,7 +261,7 @@ def generate_pip_install(pip, command, packages=None):
     if not options:
         return None
 
-    return [pip, 'install', '--disable-pip-version-check', '-c', constraints] + options
+    return pip + ['install', '--disable-pip-version-check', '-c', constraints] + options
 
 
 def command_shell(args):
diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py
index a773e917fb..2517239b3b 100644
--- a/test/runner/lib/sanity/__init__.py
+++ b/test/runner/lib/sanity/__init__.py
@@ -221,7 +221,11 @@ class SanityCodeSmellTest(SanityTest):
         :type targets: SanityTargets
         :rtype: SanityResult
         """
-        cmd = [self.path]
+        if self.path.endswith('.py'):
+            cmd = [args.python_executable, self.path]
+        else:
+            cmd = [self.path]
+
         env = ansible_environment(args, color=False)
 
         pattern = None
diff --git a/test/runner/lib/sanity/compile.py b/test/runner/lib/sanity/compile.py
index e754448057..aa0f5fdcc6 100644
--- a/test/runner/lib/sanity/compile.py
+++ b/test/runner/lib/sanity/compile.py
@@ -16,6 +16,7 @@ from lib.util import (
     SubprocessError,
     run_command,
     display,
+    find_python,
 )
 
 from lib.config import (
@@ -50,7 +51,7 @@ class CompileTest(SanityMultipleVersion):
         if not paths:
             return SanitySkipped(self.name, python_version=python_version)
 
-        cmd = ['python%s' % python_version, 'test/sanity/compile/compile.py']
+        cmd = [find_python(python_version), 'test/sanity/compile/compile.py']
 
         data = '\n'.join(paths)
 
diff --git a/test/runner/lib/sanity/import.py b/test/runner/lib/sanity/import.py
index b6c7ede108..edccce9b54 100644
--- a/test/runner/lib/sanity/import.py
+++ b/test/runner/lib/sanity/import.py
@@ -18,6 +18,7 @@ from lib.util import (
     intercept_command,
     remove_tree,
     display,
+    find_python,
 )
 
 from lib.ansible_util import (
@@ -66,7 +67,7 @@ class ImportTest(SanityMultipleVersion):
 
         remove_tree(virtual_environment_path)
 
-        cmd = ['virtualenv', virtual_environment_path, '--python', 'python%s' % python_version, '--no-setuptools', '--no-wheel']
+        cmd = ['virtualenv', virtual_environment_path, '--python', find_python(python_version), '--no-setuptools', '--no-wheel']
 
         if not args.coverage:
             cmd.append('--no-pip')
@@ -84,8 +85,8 @@ class ImportTest(SanityMultipleVersion):
 
         # make sure coverage is available in the virtual environment if needed
         if args.coverage:
-            run_command(args, generate_pip_install('pip', 'sanity.import', packages=['setuptools']), env=env)
-            run_command(args, generate_pip_install('pip', 'sanity.import', packages=['coverage']), env=env)
+            run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['setuptools']), env=env)
+            run_command(args, generate_pip_install(['pip'], 'sanity.import', packages=['coverage']), env=env)
             run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'setuptools'], env=env)
             run_command(args, ['pip', 'uninstall', '--disable-pip-version-check', '-y', 'pip'], env=env)
 
diff --git a/test/runner/lib/sanity/pep8.py b/test/runner/lib/sanity/pep8.py
index 9544fc20ca..4ebfe939c0 100644
--- a/test/runner/lib/sanity/pep8.py
+++ b/test/runner/lib/sanity/pep8.py
@@ -15,7 +15,6 @@ from lib.util import (
     SubprocessError,
     display,
     run_command,
-    find_executable,
 )
 
 from lib.config import (
@@ -56,8 +55,8 @@ class Pep8Test(SanitySingleVersion):
         paths = sorted(i.path for i in targets.include if (os.path.splitext(i.path)[1] == '.py' or i.path.startswith('bin/')) and i.path not in skip_paths_set)
 
         cmd = [
-            'python%s' % args.python_version,
-            find_executable('pycodestyle'),
+            args.python_executable,
+            '-m', 'pycodestyle',
             '--max-line-length', '160',
             '--config', '/dev/null',
             '--ignore', ','.join(sorted(current_ignore)),
diff --git a/test/runner/lib/sanity/pylint.py b/test/runner/lib/sanity/pylint.py
index e41af68e8c..ead962fbf8 100644
--- a/test/runner/lib/sanity/pylint.py
+++ b/test/runner/lib/sanity/pylint.py
@@ -252,8 +252,8 @@ class PylintTest(SanitySingleVersion):
         load_plugins = set(self.plugin_names) - disable_plugins
 
         cmd = [
-            'python%s' % args.python_version,
-            find_executable('pylint'),
+            args.python_executable,
+            '-m', 'pylint',
             '--jobs', '0',
             '--reports', 'n',
             '--max-line-length', '160',
diff --git a/test/runner/lib/sanity/rstcheck.py b/test/runner/lib/sanity/rstcheck.py
index b1e1d9ceb2..c3113fa3e4 100644
--- a/test/runner/lib/sanity/rstcheck.py
+++ b/test/runner/lib/sanity/rstcheck.py
@@ -15,6 +15,7 @@ from lib.util import (
     SubprocessError,
     run_command,
     parse_to_dict,
+    display,
     find_executable,
 )
 
@@ -22,6 +23,10 @@ from lib.config import (
     SanityConfig,
 )
 
+UNSUPPORTED_PYTHON_VERSIONS = (
+    '2.6',
+)
+
 
 class RstcheckTest(SanitySingleVersion):
     """Sanity test using rstcheck."""
@@ -31,6 +36,10 @@ class RstcheckTest(SanitySingleVersion):
         :type targets: SanityTargets
         :rtype: SanityResult
         """
+        if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
+            display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version)
+            return SanitySkipped(self.name)
+
         with open('test/sanity/rstcheck/ignore-substitutions.txt', 'r') as ignore_fd:
             ignore_substitutions = sorted(set(ignore_fd.read().splitlines()))
 
@@ -40,8 +49,8 @@ class RstcheckTest(SanitySingleVersion):
             return SanitySkipped(self.name)
 
         cmd = [
-            'python%s' % args.python_version,
-            find_executable('rstcheck'),
+            args.python_executable,
+            '-m', 'rstcheck',
             '--report', 'warning',
             '--ignore-substitutions', ','.join(ignore_substitutions),
         ] + paths
diff --git a/test/runner/lib/sanity/validate_modules.py b/test/runner/lib/sanity/validate_modules.py
index 40c0758c15..1dcbd64a72 100644
--- a/test/runner/lib/sanity/validate_modules.py
+++ b/test/runner/lib/sanity/validate_modules.py
@@ -57,7 +57,7 @@ class ValidateModulesTest(SanitySingleVersion):
             return SanitySkipped(self.name)
 
         cmd = [
-            'python%s' % args.python_version,
+            args.python_executable,
             'test/sanity/validate-modules/validate-modules',
             '--format', 'json',
             '--arg-spec',
diff --git a/test/runner/lib/sanity/yamllint.py b/test/runner/lib/sanity/yamllint.py
index 9aed5b94ad..8f3fa275af 100644
--- a/test/runner/lib/sanity/yamllint.py
+++ b/test/runner/lib/sanity/yamllint.py
@@ -65,7 +65,7 @@ class YamllintTest(SanitySingleVersion):
         :rtype: list[SanityMessage]
         """
         cmd = [
-            'python%s' % args.python_version,
+            args.python_executable,
             'test/sanity/yamllint/yamllinter.py',
         ]
 
diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py
index c303d7dba6..a12aa334bc 100644
--- a/test/runner/lib/util.py
+++ b/test/runner/lib/util.py
@@ -63,51 +63,6 @@ def remove_file(path):
         os.remove(path)
 
 
-def find_pip(path=None, version=None):
-    """
-    :type path: str | None
-    :type version: str | None
-    :rtype: str
-    """
-    if version:
-        version_info = version.split('.')
-        python_bin = find_executable('python%s' % version, path=path)
-    else:
-        version_info = sys.version_info
-        python_bin = sys.executable
-
-    choices = (
-        'pip%s' % '.'.join(str(i) for i in version_info[:2]),
-        'pip%s' % version_info[0],
-        'pip',
-    )
-
-    pip = None
-
-    for choice in choices:
-        pip = find_executable(choice, required=False, path=path)
-
-        if pip:
-            break
-
-    if not pip:
-        raise ApplicationError('Required program not found: %s' % ', '.join(choices))
-
-    with open(pip) as pip_fd:
-        shebang = pip_fd.readline().strip()
-
-    if not shebang.startswith('#!') or ' ' in shebang:
-        raise ApplicationError('Unexpected shebang in "%s": %s' % (pip, shebang))
-
-    our_python = os.path.realpath(python_bin)
-    pip_python = os.path.realpath(shebang[2:])
-
-    if our_python != pip_python and not filecmp.cmp(our_python, pip_python, False):
-        raise ApplicationError('Current interpreter "%s" does not match "%s" interpreter "%s".' % (our_python, pip, pip_python))
-
-    return pip
-
-
 def find_executable(executable, cwd=None, path=None, required=True):
     """
     :type executable: str
@@ -160,6 +115,30 @@ def find_executable(executable, cwd=None, path=None, required=True):
     return match
 
 
+def find_python(version, path=None):
+    """
+    :type version: str
+    :type path: str | None
+    :rtype: str
+    """
+    version_info = tuple(int(n) for n in version.split('.'))
+
+    if not path and version_info == sys.version_info[:len(version_info)]:
+        python_bin = sys.executable
+    else:
+        python_bin = find_executable('python%s' % version, path=path)
+
+    return python_bin
+
+
+def generate_pip_command(python):
+    """
+    :type python: str
+    :rtype: list[str]
+    """
+    return [python, '-m', 'pip.__main__']
+
+
 def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None, path=None):
     """
     :type args: TestConfig
@@ -180,7 +159,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None
     inject_path = get_coverage_path(args)
     config_path = os.path.join(inject_path, 'injector.json')
     version = python_version or args.python_version
-    interpreter = find_executable('python%s' % version, path=path)
+    interpreter = find_python(version, path)
     coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % (
         args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)))
 
diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt
index 34d6e07f47..3fe5ae6dc2 100644
--- a/test/runner/requirements/sanity.txt
+++ b/test/runner/requirements/sanity.txt
@@ -5,7 +5,7 @@ paramiko
 pycodestyle
 pylint ; python_version >= '2.7' # pylint 1.5.3+ is required for non-buggy JSON output, but 1.4+ requires python 2.7+
 pytest
-rstcheck
+rstcheck ; python_version >= '2.7' # rstcheck requires python 2.7+
 sphinx
 virtualenv
 voluptuous
diff --git a/test/runner/test.py b/test/runner/test.py
index b26e0b7dcb..51d290aba8 100755
--- a/test/runner/test.py
+++ b/test/runner/test.py
@@ -12,8 +12,8 @@ from lib.util import (
     ApplicationError,
     display,
     raw_command,
-    find_pip,
     get_docker_completion,
+    generate_pip_command,
 )
 
 from lib.delegation import (
@@ -112,7 +112,7 @@ def parse_args():
     except ImportError:
         if '--requirements' not in sys.argv:
             raise
-        raw_command(generate_pip_install(find_pip(), 'ansible-test'))
+        raw_command(generate_pip_install(generate_pip_command(sys.executable), 'ansible-test'))
         import argparse
 
     try: