_sourcecache: Fallback to fetch source when remote has missing blobsbschubert/partial-source-cache

If a remote has some missing blobs for a source, we should not fail
abruptly but instead continue to the next remote, and, in the worst
case, fetch the source again.

2 files changed, 52 insertions, 0 deletions

diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index 03ba9a74c..b987d9b8d 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -21,6 +21,7 @@ import os
 import grpc
 
 from ._remote import BaseRemote
+from ._cas.casremote import BlobNotFound
 from .storage._casbaseddirectory import CasBasedDirectory
 from ._basecache import BaseCache
 from ._exceptions import CASError, CASRemoteError, SourceCacheError
@@ -259,6 +260,10 @@ class SourceCache(BaseCache):
 
                 source.info("Pulled source {} <- {}".format(display_key, remote))
                 return True
+            except BlobNotFound as e:
+                # Not all blobs are available on this remote
+                source.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob))
+                continue
             except CASError as e:
                 raise SourceCacheError("Failed to pull source {}: {}".format(
                     display_key, e)) from e
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 8d6d5dcf4..a5863b867 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -180,3 +180,50 @@ def test_pull_fail(cli, tmpdir, datafiles):
             res.assert_task_error(ErrorDomain.PLUGIN, None)
             assert "Remote source service ({}) does not have source {} cached".format(
                 share.repo, source._get_brief_display_key()) in res.stderr
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
+    project_dir = str(datafiles)
+    element_name, repo, ref = create_test_element(tmpdir, project_dir)
+    cache_dir = os.path.join(str(tmpdir), 'cache')
+
+    # use artifact cache for sources for now, they should work the same
+    with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share:
+        with context_with_source_cache(cli, cache_dir, share, tmpdir) as context:
+            project = Project(project_dir, context)
+            project.ensure_fully_loaded()
+
+            element = project.load_elements([element_name])[0]
+            assert not element._source_cached()
+            source = list(element.sources())[0]
+
+            cas = context.get_cascache()
+            assert not cas.contains(source._get_source_name())
+
+            # Just check that we sensibly fetch and build the element
+            res = cli.run(project=project_dir, args=['build', element_name])
+            res.assert_success()
+
+            assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != []
+
+            # get root digest of source
+            sourcecache = context.sourcecache
+            digest = sourcecache.export(source)._get_digest()
+
+            move_local_cas_to_remote_source_share(str(cache_dir), share.directory)
+
+            # Remove the cas content, only keep the proto and such around
+            shutil.rmtree(os.path.join(str(tmpdir), "sourceshare", "repo", "cas", "objects"))
+            # check the share doesn't have the object
+            assert not share.has_object(digest)
+
+            state = cli.get_element_state(project_dir, element_name)
+            assert state == 'fetch needed'
+
+            # Now fetch the source and check
+            res = cli.run(project=project_dir, args=['source', 'fetch', element_name])
+            res.assert_success()
+
+            assert ("SUCCESS Fetching from {}"
+                    .format(repo.source_config(ref=ref)['url'])) in res.stderr