Add support for multiple remote caches

This extends the 'artifacts' configuration block such that a list of
`url` mappings can be given instead of a single entry. For example:

    artifacts:
      - url: http://example.com/artifacts1
      - url: ssh://ostree@example.com/artifacts2

The OSTreeCache class is updated to set up multiple remotes and query
remote refs from all of them.

There are no automated tests for this yet.

Empty URLs ('') now raise an exception. They cause breakages internally
if we allow them through, and they can only occur if the user or our
tests are misconfiguring things somehow.

We report failure to fetch from the cache by printing a message to
stderr for now. This is because BuildStream's actual logging
functionality can't be used during frontend init -- see issue #168.

13 files changed, 317 insertions, 223 deletions

diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py
index 991e84869..7dc0ce3be 100644
--- a/buildstream/_artifactcache/__init__.py
+++ b/buildstream/_artifactcache/__init__.py
@@ -18,4 +18,4 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 
-from .artifactcache import ArtifactCache
+from .artifactcache import ArtifactCache, artifact_cache_urls_from_config_node, configured_artifact_cache_urls
diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py
index 1283b37ab..1d30b2582 100644
--- a/buildstream/_artifactcache/artifactcache.py
+++ b/buildstream/_artifactcache/artifactcache.py
@@ -21,12 +21,72 @@
 import os
 from collections import Mapping
 
-from .._exceptions import ImplError
+from .._exceptions import ImplError, LoadError, LoadErrorReason
 from .. import utils
 from .. import _yaml
 
 
-# An ArtifactCache manages artifacts
+def artifact_cache_url_from_spec(spec):
+    _yaml.node_validate(spec, ['url'])
+    url = _yaml.node_get(spec, str, 'url')
+    if len(url) == 0:
+        provenance = _yaml.node_get_provenance(spec)
+        raise LoadError(LoadErrorReason.INVALID_DATA,
+                        "{}: empty artifact cache URL".format(provenance))
+    return url
+
+
+# artifact_cache_urls_from_config_node()
+#
+# Parses the configuration of remote artifact caches from a config block.
+#
+# Args:
+#   config_node (dict): The config block, which may contain the 'artifacts' key
+#
+# Returns:
+#   A list of URLs pointing to remote artifact caches.
+#
+# Raises:
+#   LoadError, if the config block contains invalid keys.
+#
+def artifact_cache_urls_from_config_node(config_node):
+    urls = []
+
+    artifacts = config_node.get('artifacts', [])
+    if isinstance(artifacts, Mapping):
+        urls.append(artifact_cache_url_from_spec(artifacts))
+    elif isinstance(artifacts, list):
+        for spec in artifacts:
+            urls.append(artifact_cache_url_from_spec(spec))
+    else:
+        provenance = _yaml.node_get_provenance(config_node, key='artifacts')
+        raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
+                              "%s: 'artifacts' must be a single 'url:' mapping, or a list of mappings" %
+                              (str(provenance)))
+    return urls
+
+
+# configured_artifact_cache_urls():
+#
+# Return the list of configured artifact remotes for a given project, in priority
+# order. This takes into account the user and project configuration.
+#
+# Args:
+#     context (Context): The BuildStream context
+#     project (Project): The BuildStream project
+#
+# Returns:
+#   A list of URLs pointing to remote artifact caches.
+#
+def configured_artifact_cache_urls(context, project):
+    project_overrides = context._get_overrides(project.name)
+    project_extra_urls = artifact_cache_urls_from_config_node(project_overrides)
+
+    return list(utils._deduplicate(
+        project_extra_urls + project.artifact_urls + context.artifact_urls))
+
+
+# An ArtifactCache manages artifacts.
 #
 # Args:
 #     context (Context): The BuildStream context
@@ -36,42 +96,24 @@ class ArtifactCache():
     def __init__(self, context, project):
 
         self.context = context
+        self.project = project
 
         os.makedirs(context.artifactdir, exist_ok=True)
         self.extractdir = os.path.join(context.artifactdir, 'extract')
 
         self._local = False
+        self.urls = []
 
-        project_overrides = context._get_overrides(project.name)
-        artifact_overrides = _yaml.node_get(project_overrides, Mapping, 'artifacts', default_value={})
-        override_url = _yaml.node_get(artifact_overrides, str, 'url', default_value='') or None
-
-        _yaml.node_validate(artifact_overrides, ['url'])
-
-        if override_url:
-            self.url = override_url
-        elif project.artifact_url:
-            self.url = project.artifact_url
-        else:
-            self.url = context.artifact_url
-
-        if self.url:
-            if self.url.startswith('/') or self.url.startswith('file://'):
-                self._local = True
-
-            self.remote = utils.url_directory_name(self.url)
-        else:
-            self.remote = None
-
-        self._offline = False
-
-    # initialize_remote():
+    # set_remotes():
     #
-    # Initialize any remote artifact cache, if needed. This may require network
-    # access and could block for several seconds.
+    # Set the list of remote caches, which is initially empty. This will
+    # contact each remote cache.
     #
-    def initialize_remote(self):
-        pass
+    # Args:
+    #     urls (list): List of artifact remote URLs, in priority order.
+    #     on_failure (callable): Called if we fail to contact one of the caches.
+    def set_remotes(self, urls, on_failure=None):
+        self.urls = urls
 
     # contains():
     #
@@ -120,37 +162,28 @@ class ArtifactCache():
         raise ImplError("Cache '{kind}' does not implement commit()"
                         .format(kind=type(self).__name__))
 
-    # set_offline()
+    # has_fetch_remotes():
     #
-    # Do not attempt to pull or push artifacts.
+    # Check whether any remote repositories are available for fetching.
     #
-    def set_offline(self):
-        self._offline = True
-
-    # can_fetch():
+    # Returns: True if any remote repositories are configured, False otherwise
     #
-    # Check whether remote repository is available for fetching.
-    #
-    # Returns: True if remote repository is available, False otherwise
-    #
-    def can_fetch(self):
-        return (not self._offline or self._local) and \
-            self.remote is not None
+    def has_fetch_remotes(self):
+        return (len(self.urls) > 0)
 
-    # can_push():
+    # has_push_remotes():
     #
-    # Check whether remote repository is available for pushing.
+    # Check whether any remote repositories are available for pushing.
     #
-    # Returns: True if remote repository is available, False otherwise
+    # Returns: True if any remote repository is configured, False otherwise
     #
-    def can_push(self):
-        return (not self._offline or self._local) and \
-            self.url is not None
+    def has_push_remotes(self):
+        return (len(self.urls) > 0)
 
     # remote_contains_key():
     #
     # Check whether the artifact for the specified Element is already available
-    # in the remote artifact cache.
+    # in any remote artifact cache.
     #
     # Args:
     #     element (Element): The Element to check
@@ -160,6 +193,3 @@ class ArtifactCache():
     #
     def remote_contains(self, element, strength=None):
         return False
-
-    def fetch_remote_refs(self):
-        pass
diff --git a/buildstream/_artifactcache/ostreecache.py b/buildstream/_artifactcache/ostreecache.py
index 78796dc6c..9987746b0 100644
--- a/buildstream/_artifactcache/ostreecache.py
+++ b/buildstream/_artifactcache/ostreecache.py
@@ -48,7 +48,7 @@ def buildref(element, key):
     return '{0}/{1}/{2}'.format(project.name, element_name, key)
 
 
-# An ArtifactCache manages artifacts in an OSTree repository
+# An OSTreeCache manages artifacts in an OSTree repository
 #
 # Args:
 #     context (Context): The BuildStream context
@@ -70,35 +70,19 @@ class OSTreeCache(ArtifactCache):
         self.repo = _ostree.ensure(ostreedir, False)
 
         self.push_url = None
-        self.pull_url = None
-
-        self._remote_refs = None
-
-    def initialize_remote(self):
-        if self.url is not None:
-            if self.url.startswith('ssh://'):
-                self.push_url = self.url
-                try:
-                    # Contact the remote cache.
-                    self.pull_url = initialize_push_connection(self.push_url)
-                except PushException as e:
-                    raise ArtifactError("BuildStream did not connect succesfully "
-                                        "to the shared cache: {}".format(e))
-            elif self.url.startswith('http://') or self.url.startswith('https://'):
-                self.push_url = None
-                self.pull_url = self.url
-            elif self._local:
-                self.push_url = self.url
-                self.pull_url = self.url
-            else:
-                raise ArtifactError("Unsupported URL scheme: {}".format(self.url))
+        self.pull_urls = []
+        self._remote_refs = {}
+
+    def set_remotes(self, urls, on_failure=None):
+        self.urls = urls
+
+        self._initialize_remotes(on_failure)
 
-            _ostree.configure_remote(self.repo, self.remote, self.pull_url)
+    def has_fetch_remotes(self):
+        return (len(self.pull_urls) > 0)
 
-    def can_push(self):
-        if self.enable_push:
-            return (not self._offline or self._local) and self.push_url is not None
-        return False
+    def has_push_remotes(self):
+        return (self.push_url is not None)
 
     # contains():
     #
@@ -134,7 +118,7 @@ class OSTreeCache(ArtifactCache):
     # Returns: True if the artifact is in the cache, False otherwise
     #
     def remote_contains_key(self, element, key):
-        if not self._remote_refs:
+        if len(self._remote_refs) == 0:
             return False
 
         ref = buildref(element, key)
@@ -241,7 +225,7 @@ class OSTreeCache(ArtifactCache):
 
     # pull():
     #
-    # Pull artifact from remote repository.
+    # Pull artifact from one of the configured remote repositories.
     #
     # Args:
     #     element (Element): The Element whose artifact is to be fetched
@@ -249,21 +233,13 @@ class OSTreeCache(ArtifactCache):
     #
     def pull(self, element, progress=None):
 
-        if self._offline and not self._local:
-            raise ArtifactError("Attempt to pull artifact while offline")
-
-        if self.pull_url.startswith("/"):
-            remote = "file://" + self.pull_url
-        else:
-            remote = self.remote
-
+        ref = buildref(element, element._get_cache_key())
         weak_ref = buildref(element, element._get_cache_key(strength=_KeyStrength.WEAK))
 
         try:
-            if self.remote_contains(element, strength=_KeyStrength.STRONG):
+            if ref in self._remote_refs:
                 # fetch the artifact using the strong cache key
-                ref = buildref(element, element._get_cache_key())
-                _ostree.fetch(self.repo, remote=remote,
+                _ostree.fetch(self.repo, remote=self._remote_refs[ref],
                               ref=ref, progress=progress)
 
                 # resolve ref to checksum
@@ -271,9 +247,9 @@ class OSTreeCache(ArtifactCache):
 
                 # update weak ref by pointing it to this newly fetched artifact
                 _ostree.set_ref(self.repo, weak_ref, rev)
-            elif self.remote_contains(element):
+            elif weak_ref in self._remote_refs:
                 # fetch the artifact using the weak cache key
-                _ostree.fetch(self.repo, remote=remote,
+                _ostree.fetch(self.repo, remote=self._remote_refs[weak_ref],
                               ref=weak_ref, progress=progress)
 
                 # resolve weak_ref to checksum
@@ -292,35 +268,6 @@ class OSTreeCache(ArtifactCache):
             raise ArtifactError("Failed to pull artifact for element {}: {}"
                                 .format(element.name, e)) from e
 
-    # fetch_remote_refs():
-    #
-    # Fetch list of artifacts from remote repository.
-    #
-    def fetch_remote_refs(self):
-        if self.pull_url.startswith("/"):
-            remote = "file://" + self.pull_url
-        elif self.remote is not None:
-            remote = self.remote
-        else:
-            raise ArtifactError("Attempt to fetch remote refs without any pull URL")
-
-        def child_action(repo, remote, q):
-            try:
-                q.put((True, _ostree.list_remote_refs(self.repo, remote=remote)))
-            except OSTreeError as e:
-                q.put((False, e))
-
-        q = multiprocessing.Queue()
-        p = multiprocessing.Process(target=child_action, args=(self.repo, remote, q))
-        p.start()
-        ret, res = q.get()
-        p.join()
-
-        if ret:
-            self._remote_refs = res
-        else:
-            raise ArtifactError("Failed to fetch remote refs") from res
-
     # push():
     #
     # Push committed artifact to remote repository.
@@ -336,17 +283,14 @@ class OSTreeCache(ArtifactCache):
     #   (ArtifactError): if there was an error
     def push(self, element):
 
-        if self._offline and not self._local:
-            raise ArtifactError("Attempt to push artifact while offline")
-
         if self.push_url is None:
             raise ArtifactError("The protocol in use does not support pushing.")
 
         ref = buildref(element, element._get_cache_key_from_artifact())
         weak_ref = buildref(element, element._get_cache_key(strength=_KeyStrength.WEAK))
-        if self.push_url.startswith("/"):
+        if self.push_url.startswith("file://"):
             # local repository
-            push_repo = _ostree.ensure(self.push_url, True)
+            push_repo = _ostree.ensure(self.push_url[7:], True)
             _ostree.fetch(push_repo, remote=self.repo.get_path().get_uri(), ref=ref)
             _ostree.fetch(push_repo, remote=self.repo.get_path().get_uri(), ref=weak_ref)
 
@@ -377,9 +321,117 @@ class OSTreeCache(ArtifactCache):
 
                 return pushed
 
-    # set_offline():
+    # _initialize_remote():
+    #
+    # Do protocol-specific initialization necessary to use a given OSTree
+    # remote.
+    #
+    # The SSH protocol that we use only supports pushing so initializing these
+    # involves contacting the remote to find out the corresponding pull URL.
+    #
+    # Args:
+    #     url (str): URL of the remote
+    #
+    # Returns:
+    #     (str, str): the pull URL and push URL for the remote
+    #
+    # Raises:
+    #     ArtifactError: if there was an error
+    def _initialize_remote(self, url):
+        if url.startswith('ssh://'):
+            try:
+                push_url = url
+                pull_url = initialize_push_connection(url)
+            except PushException as e:
+                raise ArtifactError("BuildStream did not connect successfully "
+                                    "to the shared cache {}: {}".format(url, e))
+        elif url.startswith('/'):
+            push_url = pull_url = 'file://' + url
+        elif url.startswith('file://'):
+            push_url = pull_url = url
+        elif url.startswith('http://') or url.startswith('https://'):
+            push_url = None
+            pull_url = url
+        else:
+            raise ArtifactError("Unsupported URL: {}".format(url))
+
+        return push_url, pull_url
+
+    # _ensure_remote():
     #
-    # Do not attempt to pull or push artifacts.
+    # Ensure that our OSTree repo has a remote configured for the given URL.
+    # Note that SSH access to remotes is not handled by libostree itself.
     #
-    def set_offline(self):
-        self._offline = True
+    # Args:
+    #     repo (OSTree.Repo): an OSTree repository
+    #     pull_url (str): the URL where libostree can pull from the remote
+    #
+    # Returns:
+    #     (str): the name of the remote, which can be passed to various other
+    #            operations implemented by the _ostree module.
+    #
+    # Raises:
+    #     OSTreeError: if there was a problem reported by libostree
+    def _ensure_remote(self, repo, pull_url):
+        remote_name = utils.url_directory_name(pull_url)
+        _ostree.configure_remote(repo, remote_name, pull_url)
+        return remote_name
+
+    def _initialize_remotes(self, on_failure=None):
+        self.push_url = None
+        self.pull_urls = []
+        self._remote_refs = {}
+
+        # Callback to initialize one remote in a 'multiprocessing' subprocess.
+        #
+        # We cannot do this in the main process because of the way the tasks
+        # run by the main scheduler calls into libostree using
+        # fork()-without-exec() subprocesses. OSTree fetch operations in
+        # subprocesses hang if fetch operations were previously done in the
+        # main process.
+        #
+        def child_action(url, q):
+            try:
+                push_url, pull_url = self._initialize_remote(url)
+                remote = self._ensure_remote(self.repo, pull_url)
+                remote_refs = _ostree.list_remote_refs(self.repo, remote=remote)
+                q.put((None, push_url, pull_url, remote_refs))
+            except Exception as e:
+                # Exceptions aren't automatically propagated by
+                # multiprocessing, so we catch everything here. Note that
+                # GLib.Error subclasses can't be returned (as they don't
+                # 'pickle') and so they'll be ignored.
+                q.put((e, None, None, None))
+
+        # Kick off all the initialization jobs one by one.
+        #
+        # Note that we cannot use multiprocessing.Pool here because it's not
+        # possible to pickle local functions such as child_action().
+        #
+        q = multiprocessing.Queue()
+        for url in self.urls:
+            p = multiprocessing.Process(target=child_action, args=(url, q))
+            p.start()
+            exception, push_url, pull_url, remote_refs = q.get()
+            p.join()
+
+            if exception and on_failure:
+                on_failure(url, exception)
+            elif exception:
+                raise ArtifactError() from exception
+            else:
+                # Use first pushable remote we find for pushing.
+                if push_url and not self.push_url:
+                    self.push_url = push_url
+
+                # Use all pullable remotes, in priority order
+                if pull_url and pull_url not in self.pull_urls:
+                    self.pull_urls.append(pull_url)
+
+                # Update our overall map of remote refs with any refs that are
+                # present in the new remote and were not already found in
+                # higher priority ones.
+                remote = self._ensure_remote(self.repo, pull_url)
+                for ref in remote_refs:
+                    if ref not in self._remote_refs:
+                        self._remote_refs[ref] = remote
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 6d0a19bde..78ec9f8b2 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -29,6 +29,7 @@ from . import _yaml
 from ._exceptions import LoadError, LoadErrorReason, BstError
 from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
+from ._artifactcache import artifact_cache_urls_from_config_node
 
 
 # Context()
@@ -61,8 +62,8 @@ class Context():
         # The local binary artifact cache directory
         self.artifactdir = None
 
-        # The URL from which to push and pull prebuilt artifacts
-        self.artifact_url = None
+        # The URLs from which to push and pull prebuilt artifacts
+        self.artifact_urls = []
 
         # The directory to store build logs
         self.logdir = None
@@ -161,9 +162,7 @@ class Context():
             setattr(self, dir, path)
 
         # Load artifact share configuration
-        artifacts = _yaml.node_get(defaults, Mapping, 'artifacts')
-        _yaml.node_validate(artifacts, ['url'])
-        self.artifact_url = _yaml.node_get(artifacts, str, 'url', default_value='') or None
+        self.artifact_urls = artifact_cache_urls_from_config_node(defaults)
 
         # Load logging config
         logging = _yaml.node_get(defaults, Mapping, 'logging')
diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index c9f0389d1..165556365 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -357,4 +357,4 @@ def list_remote_refs(repo, remote="origin"):
         _, refs = repo.remote_list_refs(remote)
         return refs
     except GLib.GError as e:
-        raise OSTreeError("Failed to fetch remote refs from '{}': {}".format(remote, e.message)) from e
+        raise OSTreeError(message=e.message) from e
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index cb7e0ed5e..db4c7fa85 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -40,6 +40,7 @@ from . import Scope
 from . import _site
 from . import utils
 from ._platform import Platform
+from ._artifactcache import configured_artifact_cache_urls
 
 from ._scheduler import SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
 
@@ -145,8 +146,8 @@ class Pipeline():
 
         self.initialize_workspaces()
 
-        if use_remote_cache and self.artifacts.can_fetch():
-            self.fetch_remote_refs()
+        if use_remote_cache:
+            self.initialize_remote_caches()
 
         self.resolve_cache_keys(inconsistent)
 
@@ -174,14 +175,13 @@ class Pipeline():
 
                 self.project._set_workspace(element, source, workspace)
 
-    def fetch_remote_refs(self):
-        with self.timed_activity("Fetching remote refs", silent_nested=True):
-            try:
-                self.artifacts.initialize_remote()
-                self.artifacts.fetch_remote_refs()
-            except ArtifactError:
-                self.message(MessageType.WARN, "Failed to fetch remote refs")
-                self.artifacts.set_offline()
+    def initialize_remote_caches(self):
+        def remote_failed(url, error):
+            self.message(MessageType.WARN, "Failed to fetch remote refs from {}: {}\n".format(url, error))
+
+        with self.timed_activity("Initializing remote caches", silent_nested=True):
+            artifact_urls = configured_artifact_cache_urls(self.context, self.project)
+            self.artifacts.set_remotes(artifact_urls, on_failure=remote_failed)
 
     def resolve_cache_keys(self, inconsistent):
         if inconsistent:
@@ -446,12 +446,12 @@ class Pipeline():
         if track_plan:
             track = TrackQueue(save=save)
             queues.append(track)
-        if self.artifacts.can_fetch():
+        if self.artifacts.has_fetch_remotes():
             pull = PullQueue()
             queues.append(pull)
         queues.append(fetch)
         queues.append(build)
-        if self.artifacts.can_push():
+        if self.artifacts.has_push_remotes():
             push = PushQueue()
             queues.append(push)
 
@@ -689,8 +689,8 @@ class Pipeline():
     #
     def pull(self, scheduler, elements):
 
-        if not self.artifacts.can_fetch():
-            raise PipelineError("Not configured for pulling artifacts")
+        if not self.artifacts.has_fetch_remotes():
+            raise PipelineError("Not artifact caches available for pulling artifacts")
 
         plan = elements
         self.assert_consistent(plan)
@@ -727,8 +727,8 @@ class Pipeline():
     #
     def push(self, scheduler, elements):
 
-        if not self.artifacts.can_push():
-            raise PipelineError("Not configured for pushing artifacts")
+        if not self.artifacts.has_push_remotes():
+            raise PipelineError("No artifact caches available for pushing artifacts")
 
         plan = elements
         self.assert_consistent(plan)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 50f5d9c57..35873fb72 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -28,6 +28,7 @@ from . import _yaml
 from ._profile import Topics, profile_start, profile_end
 from ._exceptions import LoadError, LoadErrorReason
 from ._options import OptionPool
+from ._artifactcache import artifact_cache_urls_from_config_node
 
 
 # The base BuildStream format version
@@ -172,9 +173,7 @@ class Project():
         #
 
         # Load artifacts pull/push configuration for this project
-        artifacts = _yaml.node_get(config, Mapping, 'artifacts', default_value={})
-        _yaml.node_validate(artifacts, ['url'])
-        self.artifact_url = _yaml.node_get(artifacts, str, 'url', default_value='') or None
+        self.artifact_urls = artifact_cache_urls_from_config_node(config)
 
         # Workspace configurations
         self._workspaces = self._load_workspace_config()
diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml
index f43989dcc..edd74ef3f 100644
--- a/buildstream/data/userconfig.yaml
+++ b/buildstream/data/userconfig.yaml
@@ -48,14 +48,6 @@ scheduler:
   #
   on-error: quit
 
-#
-#    Artifacts
-#
-artifacts:
-
-  # A url from which to push and pull prebuilt artifacts.
-  # Some protocols only support pushing.
-  url: ''
 
 #
 #    Logging
diff --git a/buildstream/utils.py b/buildstream/utils.py
index a811fe2f1..caa02135c 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -34,6 +34,7 @@ import stat
 import string
 import subprocess
 import tempfile
+import itertools
 from contextlib import contextmanager
 
 import pkg_resources
@@ -994,3 +995,32 @@ def _glob2re(pat):
         else:
             res = res + re.escape(c)
     return res + '\Z(?ms)'
+
+
+# _deduplicate()
+#
+# Remove duplicate entries in a list or other iterable.
+#
+# Copied verbatim from the unique_everseen() example at
+# https://docs.python.org/3/library/itertools.html#itertools-recipes
+#
+# Args:
+#    iterable (iterable): What to deduplicate
+#    key (callable): Optional function to map from list entry to value
+#
+# Returns:
+#    (generator): Generator that produces a deduplicated version of 'iterable'
+#
+def _deduplicate(iterable, key=None):
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        for element in itertools.filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 5f06e550d..18a4b4654 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -1,7 +1,7 @@
 import os
 import shutil
 import pytest
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, configure_remote_caches
 from tests.testutils.site import IS_LINUX
 
 from buildstream import _yaml
@@ -29,9 +29,9 @@ def assert_shared(cli, share, project, element_name):
 @pytest.mark.parametrize(
     'override_url, project_url, user_url',
     [
-        pytest.param('', '', 'share.repo', id='user-config'),
-        pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'),
-        pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'),
+        pytest.param(None, None, 'share.repo', id='user-config'),
+        pytest.param(None, 'share.repo', None, id='project-config'),
+        pytest.param('share.repo', None, None, id='project-override-in-user-config'),
     ])
 @pytest.mark.datafiles(DATA_DIR)
 def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
@@ -49,29 +49,8 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
     project_url = share.repo if project_url == 'share.repo' else project_url
     user_url = share.repo if user_url == 'share.repo' else user_url
 
-    # Configure artifact share
-    cli.configure({
-        'artifacts': {
-            'url': user_url,
-        },
-        'projects': {
-            'test': {
-                'artifacts': {
-                    'url': override_url,
-                }
-            }
-        }
-    })
-
-    if project_url:
-        project_conf_file = str(datafiles.join('project.conf'))
-        project_config = _yaml.load(project_conf_file)
-        project_config.update({
-            'artifacts': {
-                'url': project_url,
-            }
-        })
-        _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+    project_conf_file = str(datafiles.join('project.conf'))
+    configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
 
     # Now try bst push
     result = cli.run(project=project, args=['push', 'import-bin.bst'])
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index b5eddf81c..9d897a8e5 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, configure_remote_caches
 from tests.testutils.site import IS_LINUX
 
 from buildstream import _yaml
@@ -28,9 +28,9 @@ def assert_shared(cli, share, project, element_name):
 @pytest.mark.parametrize(
     'override_url, project_url, user_url',
     [
-        pytest.param('', '', 'share.repo', id='user-config'),
-        pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'),
-        pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'),
+        pytest.param(None, None, 'share.repo', id='user-config'),
+        pytest.param(None, 'share.repo', None, id='project-config'),
+        pytest.param('share.repo', None, None, id='project-override-in-user-config'),
     ])
 @pytest.mark.datafiles(DATA_DIR)
 def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
@@ -48,29 +48,8 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
     project_url = share.repo if project_url == 'share.repo' else project_url
     user_url = share.repo if user_url == 'share.repo' else user_url
 
-    # Configure artifact share
-    cli.configure({
-        'artifacts': {
-            'url': user_url,
-        },
-        'projects': {
-            'test': {
-                'artifacts': {
-                    'url': override_url,
-                }
-            }
-        }
-    })
-
-    if project_url:
-        project_conf_file = str(datafiles.join('project.conf'))
-        project_config = _yaml.load(project_conf_file)
-        project_config.update({
-            'artifacts': {
-                'url': project_url,
-            }
-        })
-        _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+    project_conf_file = str(datafiles.join('project.conf'))
+    configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
 
     # Now try bst push
     result = cli.run(project=project, args=['push', 'target.bst'])
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 9fc450a28..f0eb171c3 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -1,3 +1,3 @@
 from .runcli import cli
 from .repo import create_repo, ALL_REPO_KINDS
-from .artifactshare import create_artifact_share
+from .artifactshare import create_artifact_share, configure_remote_caches
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 907ed7671..ebf38f34b 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -3,6 +3,8 @@ import pytest
 import subprocess
 import os
 
+from buildstream import _yaml
+
 from .site import HAVE_OSTREE_CLI
 
 
@@ -106,3 +108,35 @@ class ArtifactShare():
 def create_artifact_share(directory):
 
     return ArtifactShare(directory)
+
+
+# Write out cache configuration into the user config and project config files.
+#
+# User config is set through a helper on the 'cli' object, while the
+# project.conf file is updated manually using the _yaml module.
+#
+def configure_remote_caches(cli, project_conf_file, override_url, project_url=None, user_url=None):
+    user_config = {}
+    if user_url is not None:
+        user_config['artifacts'] = {
+            'url': user_url
+        }
+
+    if override_url is not None:
+        user_config['projects'] = {
+            'test': {
+                'artifacts': {
+                    'url': override_url,
+                }
+            }
+        }
+    cli.configure(user_config)
+
+    if project_url is not None:
+        project_config = _yaml.load(project_conf_file)
+        project_config.update({
+            'artifacts': {
+                'url': project_url,
+            }
+        })
+        _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)