diff options
| -rw-r--r-- | src/buildstream/_sourcecache.py | 5 | ||||
| -rw-r--r-- | tests/sourcecache/fetch.py | 47 |
2 files changed, 52 insertions, 0 deletions
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index 03ba9a74c..b987d9b8d 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -21,6 +21,7 @@ import os import grpc from ._remote import BaseRemote +from ._cas.casremote import BlobNotFound from .storage._casbaseddirectory import CasBasedDirectory from ._basecache import BaseCache from ._exceptions import CASError, CASRemoteError, SourceCacheError @@ -259,6 +260,10 @@ class SourceCache(BaseCache): source.info("Pulled source {} <- {}".format(display_key, remote)) return True + except BlobNotFound as e: + # Not all blobs are available on this remote + source.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) + continue except CASError as e: raise SourceCacheError("Failed to pull source {}: {}".format( display_key, e)) from e diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index 8d6d5dcf4..a5863b867 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -180,3 +180,50 @@ def test_pull_fail(cli, tmpdir, datafiles): res.assert_task_error(ErrorDomain.PLUGIN, None) assert "Remote source service ({}) does not have source {} cached".format( share.repo, source._get_brief_display_key()) in res.stderr + + +@pytest.mark.datafiles(DATA_DIR) +def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): + project_dir = str(datafiles) + element_name, repo, ref = create_test_element(tmpdir, project_dir) + cache_dir = os.path.join(str(tmpdir), 'cache') + + # use artifact cache for sources for now, they should work the same + with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: + with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: + project = Project(project_dir, context) + project.ensure_fully_loaded() + + element = project.load_elements([element_name])[0] + assert not element._source_cached() + source = list(element.sources())[0] + + cas = context.get_cascache() + assert not cas.contains(source._get_source_name()) + + # Just check that we sensibly fetch and build the element + res = cli.run(project=project_dir, args=['build', element_name]) + res.assert_success() + + assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != [] + + # get root digest of source + sourcecache = context.sourcecache + digest = sourcecache.export(source)._get_digest() + + move_local_cas_to_remote_source_share(str(cache_dir), share.directory) + + # Remove the cas content, only keep the proto and such around + shutil.rmtree(os.path.join(str(tmpdir), "sourceshare", "repo", "cas", "objects")) + # check the share doesn't have the object + assert not share.has_object(digest) + + state = cli.get_element_state(project_dir, element_name) + assert state == 'fetch needed' + + # Now fetch the source and check + res = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + res.assert_success() + + assert ("SUCCESS Fetching from {}" + .format(repo.source_config(ref=ref)['url'])) in res.stderr |