[stable-2.8] Use AZP config for integration-aliases test.

No changelog entry since this test is limited to the ansible/ansible repo.

Backport of https://github.com/ansible/ansible/pull/72842

2 files changed, 91 insertions, 38 deletions

diff --git a/test/runner/lib/sanity/integration_aliases.py b/test/runner/lib/sanity/integration_aliases.py
index 9ece2599e2..f96455f3a7 100644
--- a/test/runner/lib/sanity/integration_aliases.py
+++ b/test/runner/lib/sanity/integration_aliases.py
@@ -3,9 +3,10 @@ from __future__ import absolute_import, print_function
 
 import json
 import textwrap
-import re
 import os
 
+import lib.types as t
+
 from lib.sanity import (
     SanitySingleVersion,
     SanityMessage,
@@ -32,12 +33,15 @@ from lib.cloud import (
 
 from lib.util import (
     display,
+    find_python,
+    raw_command,
 )
 
 
 class IntegrationAliasesTest(SanitySingleVersion):
     """Sanity test to evaluate integration test aliases."""
-    SHIPPABLE_YML = 'shippable.yml'
+    CI_YML = '.azure-pipelines/azure-pipelines.yml'
+    TEST_ALIAS_PREFIX = 'shippable'  # this will be changed at some point in the future
 
     DISABLED = 'disabled/'
     UNSTABLE = 'unstable/'
@@ -80,63 +84,96 @@ class IntegrationAliasesTest(SanitySingleVersion):
     def __init__(self):
         super(IntegrationAliasesTest, self).__init__()
 
-        self._shippable_yml_lines = []  # type: list[str]
-        self._shippable_test_groups = {}  # type: dict[str, set[int]]
+        self._ci_config = {}  # type: t.Dict[str, t.Any]
+        self._ci_test_groups = {}  # type: t.Dict[str, t.List[int]]
 
-    @property
-    def shippable_yml_lines(self):
-        """
-        :rtype: list[str]
-        """
-        if not self._shippable_yml_lines:
-            with open(self.SHIPPABLE_YML, 'r') as shippable_yml_fd:
-                self._shippable_yml_lines = shippable_yml_fd.read().splitlines()
+    def load_ci_config(self, args):  # type: (SanityConfig) -> t.Dict[str, t.Any]
+        """Load and return the CI YAML configuration."""
+        if not self._ci_config:
+            self._ci_config = self.load_yaml(args, self.CI_YML)
 
-        return self._shippable_yml_lines
+        return self._ci_config
 
     @property
-    def shippable_test_groups(self):
-        """
-        :rtype: dict[str, set[int]]
-        """
-        if not self._shippable_test_groups:
-            matches = [re.search(r'^[ #]+- env: T=(?P<group>[^/]+)/(?P<params>.+)/(?P<number>[1-9])$', line) for line in self.shippable_yml_lines]
-            entries = [(match.group('group'), int(match.group('number'))) for match in matches if match]
+    def ci_test_groups(self):  # type: () -> t.Dict[str, t.List[int]]
+        """Return a dictionary of CI test names and their group(s)."""
+        if not self._ci_test_groups:
+            test_groups = {}
+
+            for stage in self._ci_config['stages']:
+                for job in stage['jobs']:
+                    if job.get('template') != 'templates/matrix.yml':
+                        continue
+
+                    parameters = job['parameters']
+
+                    groups = parameters.get('groups', [])
+                    test_format = parameters.get('testFormat', '{0}')
+                    test_group_format = parameters.get('groupFormat', '{0}/{{1}}')
 
-            for key, value in entries:
-                if key not in self._shippable_test_groups:
-                    self._shippable_test_groups[key] = set()
+                    for target in parameters['targets']:
+                        test = target.get('test') or target.get('name')
 
-                self._shippable_test_groups[key].add(value)
+                        if groups:
+                            tests_formatted = [test_group_format.format(test_format).format(test, group) for group in groups]
+                        else:
+                            tests_formatted = [test_format.format(test)]
 
-        return self._shippable_test_groups
+                        for test_formatted in tests_formatted:
+                            parts = test_formatted.split('/')
+                            key = parts[0]
 
-    def format_shippable_group_alias(self, name, fallback=''):
+                            if key in ('sanity', 'units'):
+                                continue
+
+                            try:
+                                group = int(parts[-1])
+                            except ValueError:
+                                continue
+
+                            if group < 1 or group > 99:
+                                continue
+
+                            group_set = test_groups.setdefault(key, set())
+                            group_set.add(group)
+
+            self._ci_test_groups = dict((key, sorted(value)) for key, value in test_groups.items())
+
+        return self._ci_test_groups
+
+    def format_test_group_alias(self, name, fallback=''):
         """
         :type name: str
         :type fallback: str
         :rtype: str
         """
-        group_numbers = self.shippable_test_groups.get(name, None)
+        group_numbers = self.ci_test_groups.get(name, None)
 
         if group_numbers:
             if min(group_numbers) != 1:
-                display.warning('Min test group "%s" in shippable.yml is %d instead of 1.' % (name, min(group_numbers)), unique=True)
+                display.warning('Min test group "%s" in %s is %d instead of 1.' % (name, self.CI_YML, min(group_numbers)), unique=True)
 
             if max(group_numbers) != len(group_numbers):
-                display.warning('Max test group "%s" in shippable.yml is %d instead of %d.' % (name, max(group_numbers), len(group_numbers)), unique=True)
+                display.warning('Max test group "%s" in %s is %d instead of %d.' % (name, self.CI_YML, max(group_numbers), len(group_numbers)), unique=True)
 
             if len(group_numbers) > 1:
-                alias = 'shippable/%s/group[%d-%d]/' % (name, min(group_numbers), max(group_numbers))
+                alias = '%s/%s/group[%d-%d]/' % (self.TEST_ALIAS_PREFIX, name, min(group_numbers), max(group_numbers))
             else:
-                alias = 'shippable/%s/group%d/' % (name, min(group_numbers))
+                alias = '%s/%s/group%d/' % (self.TEST_ALIAS_PREFIX, name, min(group_numbers))
         elif fallback:
-            alias = 'shippable/%s/group%d/' % (fallback, 1)
+            alias = '%s/%s/group%d/' % (self.TEST_ALIAS_PREFIX, fallback, 1)
         else:
-            raise Exception('cannot find test group "%s" in shippable.yml' % name)
+            raise Exception('cannot find test group "%s" in %s' % (name, self.CI_YML))
 
         return alias
 
+    def load_yaml(self, args, path):  # type: (SanityConfig, str) -> t.Dict[str, t.Any]
+        """Load the specified YAML file and return the contents."""
+        yaml_to_json_path = 'test/sanity/integration-aliases/yaml_to_json.py'
+        python = find_python(args.python_version)
+
+        return json.loads(raw_command([python, yaml_to_json_path], data=open(path).read(), capture=True)[0])
+
     def test(self, args, targets):
         """
         :type args: SanityConfig
@@ -146,10 +183,10 @@ class IntegrationAliasesTest(SanitySingleVersion):
         if args.explain:
             return SanitySuccess(self.name)
 
-        if not os.path.isfile(self.SHIPPABLE_YML):
+        if not os.path.isfile(self.CI_YML):
             return SanityFailure(self.name, messages=[SanityMessage(
                 message='file missing',
-                path=self.SHIPPABLE_YML,
+                path=self.CI_YML,
             )])
 
         results = dict(
@@ -157,6 +194,7 @@ class IntegrationAliasesTest(SanitySingleVersion):
             labels={},
         )
 
+        self.load_ci_config(args)
         self.check_changes(args, results)
 
         with open('test/results/bot/data-sanity-ci.json', 'w') as results_fd:
@@ -197,13 +235,13 @@ class IntegrationAliasesTest(SanitySingleVersion):
 
         messages += self.check_ci_group(
             targets=tuple(filter_targets(posix_targets, ['cloud/'], include=False, directories=False, errors=False)),
-            find=self.format_shippable_group_alias('linux').replace('linux', 'posix'),
+            find=self.format_test_group_alias('linux').replace('linux', 'posix'),
         )
 
         for cloud in clouds:
             messages += self.check_ci_group(
                 targets=tuple(filter_targets(posix_targets, ['cloud/%s/' % cloud], include=True, directories=False, errors=False)),
-                find=self.format_shippable_group_alias(cloud, 'cloud'),
+                find=self.format_test_group_alias(cloud, 'cloud'),
             )
 
         return messages
@@ -218,7 +256,7 @@ class IntegrationAliasesTest(SanitySingleVersion):
 
         messages += self.check_ci_group(
             targets=windows_targets,
-            find=self.format_shippable_group_alias('windows'),
+            find=self.format_test_group_alias('windows'),
         )
 
         return messages
diff --git a/test/sanity/integration-aliases/yaml_to_json.py b/test/sanity/integration-aliases/yaml_to_json.py
new file mode 100644
index 0000000000..74a45f009f
--- /dev/null
+++ b/test/sanity/integration-aliases/yaml_to_json.py
@@ -0,0 +1,15 @@
+"""Read YAML from stdin and write JSON to stdout."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+import sys
+
+from yaml import load
+
+try:
+    from yaml import CSafeLoader as SafeLoader
+except ImportError:
+    from yaml import SafeLoader
+
+json.dump(load(sys.stdin, Loader=SafeLoader), sys.stdout)