diff options
author | Jürg Billeter <j@bitron.ch> | 2019-04-01 12:19:32 +0200 |
---|---|---|
committer | Jürg Billeter <j@bitron.ch> | 2019-04-12 11:17:33 +0200 |
commit | 18f3a8ad07746cc62319b2520752a078c9cd4dfb (patch) | |
tree | 465b2905e7b77cfd31842d8e80de9cd70ee4beaf | |
parent | a16a5cfc1b91eb09b22baff63571c88f03381dfe (diff) | |
download | buildstream-18f3a8ad07746cc62319b2520752a078c9cd4dfb.tar.gz |
_sandboxremote.py: Make output file fetching conditional
Output files need to be downloaded only if they are required in the
local cache or if artifact push remotes need them.
-rw-r--r-- | buildstream/sandbox/_sandboxremote.py | 30 |
1 files changed, 23 insertions, 7 deletions
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 46a3db527..1219afefc 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -53,6 +53,8 @@ class SandboxRemote(Sandbox): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self._output_files_required = kwargs.get('output_files_required', True) + config = kwargs['specs'] # This should be a RemoteExecutionSpec if config is None: return @@ -274,7 +276,9 @@ class SandboxRemote(Sandbox): raise SandboxError("Output directory structure had no digest attached.") context = self._get_context() + project = self._get_project() cascache = context.get_cascache() + artifactcache = context.artifactcache casremote = CASRemote(self.storage_remote_spec) # Now do a pull to ensure we have the full directory structure. @@ -289,13 +293,25 @@ class SandboxRemote(Sandbox): new_dir = CasBasedDirectory(context.artifactcache.cas, digest=dir_digest) self._set_virtual_directory(new_dir) - # Fetch the file blobs - required_blobs = cascache.required_blobs_for_directory(dir_digest) - local_missing_blobs = cascache.local_missing_blobs(required_blobs) - remote_missing_blobs = cascache.fetch_blobs(casremote, local_missing_blobs) - if remote_missing_blobs: - raise SandboxError("{} output files are missing on the CAS server" - .format(len(remote_missing_blobs))) + # Fetch the file blobs if needed + if self._output_files_required or artifactcache.has_push_remotes(): + required_blobs = cascache.required_blobs_for_directory(dir_digest) + local_missing_blobs = cascache.local_missing_blobs(required_blobs) + if local_missing_blobs: + if self._output_files_required: + # Fetch all blobs from Remote Execution CAS server + blobs_to_fetch = local_missing_blobs + else: + # Output files are not required in the local cache, + # however, artifact push remotes will need them. + # Only fetch blobs that are missing on one or multiple + # artifact servers. + blobs_to_fetch = artifactcache.find_missing_blobs(project, local_missing_blobs) + + remote_missing_blobs = cascache.fetch_blobs(casremote, blobs_to_fetch) + if remote_missing_blobs: + raise SandboxError("{} output files are missing on the CAS server" + .format(len(remote_missing_blobs))) def _run(self, command, flags, *, cwd, env): stdout, stderr = self._get_output() |