Allow splitting artifact caches

This is now split into storage/index remotes, where the former is
expected to be a CASRemote and the latter a BuildStream-specific
remote with the extensions required to store BuildStream artifact
protos.

1 files changed, 71 insertions, 6 deletions

diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index 2f235f38c..8b01a9ebe 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -6,7 +6,7 @@ import os
 
 import pytest
 
-from buildstream._remote import RemoteSpec
+from buildstream._remote import RemoteSpec, RemoteType
 from buildstream._artifactcache import ArtifactCache
 from buildstream._project import Project
 from buildstream.utils import _deduplicate
@@ -24,12 +24,28 @@ cache2 = RemoteSpec(url='https://example.com/cache2', push=False)
 cache3 = RemoteSpec(url='https://example.com/cache3', push=False)
 cache4 = RemoteSpec(url='https://example.com/cache4', push=False)
 cache5 = RemoteSpec(url='https://example.com/cache5', push=False)
-cache6 = RemoteSpec(url='https://example.com/cache6', push=True)
+cache6 = RemoteSpec(url='https://example.com/cache6',
+                    push=True,
+                    type=RemoteType.ALL)
+cache7 = RemoteSpec(url='https://index.example.com/cache1',
+                    push=True,
+                    type=RemoteType.INDEX)
+cache8 = RemoteSpec(url='https://storage.example.com/cache1',
+                    push=True,
+                    type=RemoteType.STORAGE)
 
 
 # Generate cache configuration fragments for the user config and project config files.
 #
-def configure_remote_caches(override_caches, project_caches=None, user_caches=None):
+def configure_remote_caches(override_caches,
+                            project_caches=None,
+                            user_caches=None):
+    type_strings = {
+        RemoteType.INDEX: 'index',
+        RemoteType.STORAGE: 'storage',
+        RemoteType.ALL: 'all'
+    }
+
     if project_caches is None:
         project_caches = []
 
@@ -41,10 +57,15 @@ def configure_remote_caches(override_caches, project_caches=None, user_caches=No
         user_config['artifacts'] = {
             'url': user_caches[0].url,
             'push': user_caches[0].push,
+            'type': type_strings[user_caches[0].type]
         }
     elif len(user_caches) > 1:
         user_config['artifacts'] = [
-            {'url': cache.url, 'push': cache.push} for cache in user_caches
+            {
+                'url': cache.url,
+                'push': cache.push,
+                'type': type_strings[cache.type]
+            } for cache in user_caches
         ]
 
     if len(override_caches) == 1:
@@ -53,6 +74,7 @@ def configure_remote_caches(override_caches, project_caches=None, user_caches=No
                 'artifacts': {
                     'url': override_caches[0].url,
                     'push': override_caches[0].push,
+                    'type': type_strings[override_caches[0].type]
                 }
             }
         }
@@ -60,7 +82,11 @@ def configure_remote_caches(override_caches, project_caches=None, user_caches=No
         user_config['projects'] = {
             'test': {
                 'artifacts': [
-                    {'url': cache.url, 'push': cache.push} for cache in override_caches
+                    {
+                        'url': cache.url,
+                        'push': cache.push,
+                        'type': type_strings[cache.type]
+                    } for cache in override_caches
                 ]
             }
         }
@@ -72,12 +98,17 @@ def configure_remote_caches(override_caches, project_caches=None, user_caches=No
                 'artifacts': {
                     'url': project_caches[0].url,
                     'push': project_caches[0].push,
+                    'type': type_strings[project_caches[0].type],
                 }
             })
         elif len(project_caches) > 1:
             project_config.update({
                 'artifacts': [
-                    {'url': cache.url, 'push': cache.push} for cache in project_caches
+                    {
+                        'url': cache.url,
+                        'push': cache.push,
+                        'type': type_strings[cache.type]
+                    } for cache in project_caches
                 ]
             })
 
@@ -96,6 +127,7 @@ def configure_remote_caches(override_caches, project_caches=None, user_caches=No
         pytest.param([cache1], [cache2], [cache3], id='project-override-in-user-config'),
         pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id='list-order'),
         pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id='duplicates'),
+        pytest.param([cache7, cache8], [], [cache1], id='split-caches')
     ])
 def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user_caches):
     # Produce a fake user and project config with the cache configuration.
@@ -149,3 +181,36 @@ def test_missing_certs(cli, datafiles, config_key, config_value):
     # This does not happen for a simple `bst show`.
     result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+
+
+# Assert that BuildStream complains when someone attempts to define
+# only one type of storage.
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize(
+    'override_caches, project_caches, user_caches',
+    [
+        # The leftmost cache is the highest priority one in all cases here.
+        pytest.param([], [], [cache7], id='index-user'),
+        pytest.param([], [], [cache8], id='storage-user'),
+        pytest.param([], [cache7], [], id='index-project'),
+        pytest.param([], [cache8], [], id='storage-project'),
+        pytest.param([cache7], [], [], id='index-override'),
+        pytest.param([cache8], [], [], id='storage-override'),
+    ])
+def test_only_one(cli, datafiles, override_caches, project_caches, user_caches):
+    project = os.path.join(datafiles.dirname, datafiles.basename, 'only-one')
+
+    # Produce a fake user and project config with the cache configuration.
+    user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches)
+    project_config['name'] = 'test'
+
+    cli.configure(user_config)
+
+    project_config_file = os.path.join(project, 'project.conf')
+    _yaml.roundtrip_dump(project_config, file=project_config_file)
+
+    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
+    #
+    # This does not happen for a simple `bst show`.
+    result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
+    result.assert_main_error(ErrorDomain.STREAM, None)