diff options
author | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2018-09-18 09:53:26 +0000 |
---|---|---|
committer | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2018-09-18 09:53:26 +0000 |
commit | b587579f76f2e47aa9f24a9c00075f79428be818 (patch) | |
tree | c418b704514f63ceaa3406d36420fb40093330c0 | |
parent | f2ae46f8e59212f78336cf3e4adfe4ac40095110 (diff) | |
parent | e32221b6965f0fc1c3b3e348af3ebe25d7aac46e (diff) | |
download | buildstream-b587579f76f2e47aa9f24a9c00075f79428be818.tar.gz |
Merge branch 'danielsilverstone-ct/fix-654' into 'master'
sandbox/_sandboxremote.py: Acquire cache via Platform
See merge request BuildStream/buildstream!797
-rw-r--r-- | buildstream/_artifactcache/artifactcache.py | 5 | ||||
-rw-r--r-- | buildstream/sandbox/_sandboxremote.py | 19 | ||||
-rw-r--r-- | tests/artifactcache/pull.py | 3 |
3 files changed, 12 insertions, 15 deletions
diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index eb7759d83..b5c27a165 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -91,6 +91,7 @@ class ArtifactCache(): self._cache_size = None # The current cache size, sometimes it's an estimate self._cache_quota = None # The cache quota self._cache_lower_threshold = None # The target cache size for a cleanup + self._remotes_setup = False # Check to prevent double-setup of remotes os.makedirs(self.extractdir, exist_ok=True) os.makedirs(self.tmpdir, exist_ok=True) @@ -143,6 +144,10 @@ class ArtifactCache(): # def setup_remotes(self, *, use_config=False, remote_url=None): + # Ensure we do not double-initialise since this can be expensive + assert(not self._remotes_setup) + self._remotes_setup = True + # Initialize remote artifact caches. We allow the commandline to override # the user config in some cases (for example `bst push --remote=...`). has_remote_caches = False diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 296b20351..82968ecb0 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -27,7 +27,7 @@ from . import Sandbox from ..storage._filebaseddirectory import FileBasedDirectory from ..storage._casbaseddirectory import CasBasedDirectory from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc -from .._artifactcache.cascache import CASCache +from .._platform import Platform class SandboxError(Exception): @@ -43,7 +43,6 @@ class SandboxRemote(Sandbox): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.cascache = None url = urlparse(kwargs['server_url']) if not url.scheme or not url.hostname or not url.port: @@ -56,12 +55,6 @@ class SandboxRemote(Sandbox): self.server_url = '{}:{}'.format(url.hostname, url.port) - def _get_cascache(self): - if self.cascache is None: - self.cascache = CASCache(self._get_context()) - self.cascache.setup_remotes(use_config=True) - return self.cascache - def run_remote_command(self, command, input_root_digest, working_directory, environment): # Sends an execution request to the remote execution server. # @@ -78,8 +71,8 @@ class SandboxRemote(Sandbox): output_files=[], output_directories=[self._output_directory], platform=None) - - cascache = self._get_cascache() + platform = Platform.get_platform() + cascache = platform.artifactcache # Upload the Command message to the remote CAS server command_digest = cascache.push_message(self._get_project(), remote_command) if not command_digest or not cascache.verify_digest_pushed(self._get_project(), command_digest): @@ -141,7 +134,8 @@ class SandboxRemote(Sandbox): if tree_digest is None or not tree_digest.hash: raise SandboxError("Output directory structure had no digest attached.") - cascache = self._get_cascache() + platform = Platform.get_platform() + cascache = platform.artifactcache # Now do a pull to ensure we have the necessary parts. dir_digest = cascache.pull_tree(self._get_project(), tree_digest) if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes: @@ -176,7 +170,8 @@ class SandboxRemote(Sandbox): upload_vdir.recalculate_hash() - cascache = self._get_cascache() + platform = Platform.get_platform() + cascache = platform.artifactcache # Now, push that key (without necessarily needing a ref) to the remote. vdir_digest = cascache.push_directory(self._get_project(), upload_vdir) if not vdir_digest or not cascache.verify_digest_pushed(self._get_project(), vdir_digest): diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py index 6336e7ab1..45560aabf 100644 --- a/tests/artifactcache/pull.py +++ b/tests/artifactcache/pull.py @@ -137,7 +137,6 @@ def _test_pull(user_config_file, project_dir, artifact_dir, # Manually setup the CAS remote cas.setup_remotes(use_config=True) - cas.initialize_remotes() if cas.has_push_remotes(element=element): # Push the element's artifact @@ -274,7 +273,6 @@ def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest # Manually setup the CAS remote cas.setup_remotes(use_config=True) - cas.initialize_remotes() if cas.has_push_remotes(): directory = remote_execution_pb2.Directory() @@ -310,7 +308,6 @@ def _test_pull_tree(user_config_file, project_dir, artifact_dir, artifact_digest # Manually setup the CAS remote cas.setup_remotes(use_config=True) - cas.initialize_remotes() if cas.has_push_remotes(): # Pull the artifact using the Tree object |