Improve combination generation, include whether to generate numbered suffixes

3 files changed, 22 insertions, 15 deletions

diff --git a/buildstream/_mirror.py b/buildstream/_mirror.py
index d98a5b0b2..c98529f0e 100644
--- a/buildstream/_mirror.py
+++ b/buildstream/_mirror.py
@@ -58,7 +58,7 @@ class DefaultMirror(Mirror):
     def get_mirror_uris(self, uri, source):
         url_prefix, url_body = uri.split(utils._ALIAS_SEPARATOR, 1)
         for alias_uri in self.aliases.get(url_prefix, []):
-            yield alias_uri + url_body
+            yield alias_uri + url_body, False
 
 
 class BstGeneratedMirror(Mirror):
@@ -76,4 +76,4 @@ class BstGeneratedMirror(Mirror):
         self.aliases_covered = _yaml.node_get(node, list, 'aliases-covered')
 
     def get_mirror_uris(self, uri, source):
-        yield source.get_normalised_mirror_path(uri, prefix=self.site)
+        yield source.get_normalised_mirror_path(uri, prefix=self.site), True
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 52fd36e05..56b644db1 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -217,19 +217,22 @@ class Project():
 
     # generate_alias_combinations()
     #
-    # Yields every unique combination of mirrors for each alias
+    # Yields every unique combination of mirrors for each alias, including a flag for
+    # whether a numerical suffix should be appended.
     #
     # e.g. alias 'foo' has a mirror at 'mirror-A', and the normal alias at 'upstream-A'
     #      alias 'bar' has no mirror, but does have the normal alias at 'upstream-B'
-    #      We would yield {'foo': 'mirror-A', 'bar': 'upstream-B'},
-    #      and            {'foo': 'upstream-A', 'bar': 'upstream-B'},
+    #      'mirror-A' should have a numerical suffix appended.
+    #      We would yield {'foo': ('mirror-A', True), 'bar': ('upstream-B', False)},
+    #      and            {'foo': ('upstream-A', False), 'bar': ('upstream-B', False)},
     #
     # Args:
     #    URLs (list): A list of URLs to generate combinations for if they're
     #                 prefixed with an appropriate alias.
     #
     # Yields:
-    #    a dict mapping URLs to a mirrored URL
+    #    a dict mapping URLs to a tuple of the mirrored URL and whether that URL
+    #    should have a numbered suffix appended.
     #
     def generate_alias_combinations(self, urls):
 
@@ -249,26 +252,29 @@ class Project():
         reordered_mirrors = OrderedDict(self.mirrors)
         reordered_mirrors.move_to_end(self.default_mirror, last=False)  # pylint: disable=no-member
 
-        combinations = [[]]
+        combinations = [OrderedDict()]
         for url in urls:
             new_combinations = []
             for combination in combinations:
                 alias = urls_to_aliases[url]
                 for mirror in reordered_mirrors.values():
-                    for uri in mirror.get_mirror_uris(url, self):
-                        new_combinations.append(combination + [uri])
+                    for uri, append_number in mirror.get_mirror_uris(url, self):
+                        new_combination = OrderedDict(combination)
+                        new_combination[url] = (uri, append_number)
+                        new_combinations.append(new_combination)
+
+                # Add the default aliases as valid mirrors
                 if alias in self._aliases:
                     default_alias = self._aliases[alias]
                     _, body = url.split(utils._ALIAS_SEPARATOR, 1)
                     new_url = default_alias + body
-                    new_combinations.append(combination + [new_url])
+                    new_combination = OrderedDict(combination)
+                    new_combination[url] = (new_url, False)
+                    new_combinations.append(new_combination)
             combinations = new_combinations
 
         for combination in combinations:
-            out_combination = {}
-            for i, url in enumerate(urls):
-                out_combination[url] = combination[i]
-            yield out_combination
+            yield combination
 
     # _load():
     #
diff --git a/buildstream/source.py b/buildstream/source.py
index 5f9f02a59..c0a8820d9 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -642,7 +642,8 @@ class Source(Plugin):
         context = self._get_context()
         source_kind = type(self)
         for combination in project.generate_alias_combinations(self._used_urls):
-            new_source = source_kind(context, project, self.__meta, uri_overrides=combination)
+            uri_overrides = {k: v[0] for k, v in combination.items()}
+            new_source = source_kind(context, project, self.__meta, uri_overrides=uri_overrides)
             new_source._preflight()
             try:
                 new_source._fetch()