Add support for multiple remote caches

This extends the 'artifacts' configuration block such that a list of
`url` mappings can be given instead of a single entry. For example:

    artifacts:
      - url: http://example.com/artifacts1
      - url: ssh://ostree@example.com/artifacts2

The OSTreeCache class is updated to set up multiple remotes and query
remote refs from all of them.

There are no automated tests for this yet.

Empty URLs ('') now raise an exception. They cause breakages internally
if we allow them through, and they can only occur if the user or our
tests are misconfiguring things somehow.

We report failure to fetch from the cache by printing a message to
stderr for now. This is because BuildStream's actual logging
functionality can't be used during frontend init -- see issue #168.

4 files changed, 47 insertions, 55 deletions

diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 5f06e550d..18a4b4654 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -1,7 +1,7 @@
 import os
 import shutil
 import pytest
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, configure_remote_caches
 from tests.testutils.site import IS_LINUX
 
 from buildstream import _yaml
@@ -29,9 +29,9 @@ def assert_shared(cli, share, project, element_name):
 @pytest.mark.parametrize(
     'override_url, project_url, user_url',
     [
-        pytest.param('', '', 'share.repo', id='user-config'),
-        pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'),
-        pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'),
+        pytest.param(None, None, 'share.repo', id='user-config'),
+        pytest.param(None, 'share.repo', None, id='project-config'),
+        pytest.param('share.repo', None, None, id='project-override-in-user-config'),
     ])
 @pytest.mark.datafiles(DATA_DIR)
 def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
@@ -49,29 +49,8 @@ def test_push_pull(cli, tmpdir, datafiles, override_url, project_url, user_url):
     project_url = share.repo if project_url == 'share.repo' else project_url
     user_url = share.repo if user_url == 'share.repo' else user_url
 
-    # Configure artifact share
-    cli.configure({
-        'artifacts': {
-            'url': user_url,
-        },
-        'projects': {
-            'test': {
-                'artifacts': {
-                    'url': override_url,
-                }
-            }
-        }
-    })
-
-    if project_url:
-        project_conf_file = str(datafiles.join('project.conf'))
-        project_config = _yaml.load(project_conf_file)
-        project_config.update({
-            'artifacts': {
-                'url': project_url,
-            }
-        })
-        _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+    project_conf_file = str(datafiles.join('project.conf'))
+    configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
 
     # Now try bst push
     result = cli.run(project=project, args=['push', 'import-bin.bst'])
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index b5eddf81c..9d897a8e5 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, configure_remote_caches
 from tests.testutils.site import IS_LINUX
 
 from buildstream import _yaml
@@ -28,9 +28,9 @@ def assert_shared(cli, share, project, element_name):
 @pytest.mark.parametrize(
     'override_url, project_url, user_url',
     [
-        pytest.param('', '', 'share.repo', id='user-config'),
-        pytest.param('', 'share.repo', '/tmp/share/user', id='project-config'),
-        pytest.param('share.repo', '/tmp/share/project', '/tmp/share/user', id='project-override-in-user-config'),
+        pytest.param(None, None, 'share.repo', id='user-config'),
+        pytest.param(None, 'share.repo', None, id='project-config'),
+        pytest.param('share.repo', None, None, id='project-override-in-user-config'),
     ])
 @pytest.mark.datafiles(DATA_DIR)
 def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
@@ -48,29 +48,8 @@ def test_push(cli, tmpdir, datafiles, override_url, user_url, project_url):
     project_url = share.repo if project_url == 'share.repo' else project_url
     user_url = share.repo if user_url == 'share.repo' else user_url
 
-    # Configure artifact share
-    cli.configure({
-        'artifacts': {
-            'url': user_url,
-        },
-        'projects': {
-            'test': {
-                'artifacts': {
-                    'url': override_url,
-                }
-            }
-        }
-    })
-
-    if project_url:
-        project_conf_file = str(datafiles.join('project.conf'))
-        project_config = _yaml.load(project_conf_file)
-        project_config.update({
-            'artifacts': {
-                'url': project_url,
-            }
-        })
-        _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)
+    project_conf_file = str(datafiles.join('project.conf'))
+    configure_remote_caches(cli, project_conf_file, override_url, project_url, user_url)
 
     # Now try bst push
     result = cli.run(project=project, args=['push', 'target.bst'])
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 9fc450a28..f0eb171c3 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -1,3 +1,3 @@
 from .runcli import cli
 from .repo import create_repo, ALL_REPO_KINDS
-from .artifactshare import create_artifact_share
+from .artifactshare import create_artifact_share, configure_remote_caches
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 907ed7671..ebf38f34b 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -3,6 +3,8 @@ import pytest
 import subprocess
 import os
 
+from buildstream import _yaml
+
 from .site import HAVE_OSTREE_CLI
 
 
@@ -106,3 +108,35 @@ class ArtifactShare():
 def create_artifact_share(directory):
 
     return ArtifactShare(directory)
+
+
+# Write out cache configuration into the user config and project config files.
+#
+# User config is set through a helper on the 'cli' object, while the
+# project.conf file is updated manually using the _yaml module.
+#
+def configure_remote_caches(cli, project_conf_file, override_url, project_url=None, user_url=None):
+    user_config = {}
+    if user_url is not None:
+        user_config['artifacts'] = {
+            'url': user_url
+        }
+
+    if override_url is not None:
+        user_config['projects'] = {
+            'test': {
+                'artifacts': {
+                    'url': override_url,
+                }
+            }
+        }
+    cli.configure(user_config)
+
+    if project_url is not None:
+        project_config = _yaml.load(project_conf_file)
+        project_config.update({
+            'artifacts': {
+                'url': project_url,
+            }
+        })
+        _yaml.dump(_yaml.node_sanitize(project_config), filename=project_conf_file)