From 2247917b2af1f5ab842c068897a956657178fb57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Sat, 2 Mar 2019 09:31:19 +0100 Subject: _sandboxremote.py: Use CasBasedDirectory as sandbox root This allows bypassing the local filesystem. --- buildstream/sandbox/_sandboxremote.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream/sandbox/_sandboxremote.py') diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 348ebca1b..6a0f1e6d0 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -452,6 +452,10 @@ class SandboxRemote(Sandbox): def _create_batch(self, main_group, flags, *, collect=None): return _SandboxRemoteBatch(self, main_group, flags, collect=collect) + def _use_cas_based_directory(self): + # Always use CasBasedDirectory for remote execution + return True + # _SandboxRemoteBatch() # -- cgit v1.2.1 From f96d93c6f150e904ee2bfad7f1938cfcdc5e7112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 27 Feb 2019 18:19:38 +0100 Subject: _sandboxremote.py: Remove support for FileBasedDirectory as sandbox root Remote execution should always use CasBasedDirectory. --- buildstream/sandbox/_sandboxremote.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'buildstream/sandbox/_sandboxremote.py') diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 6a0f1e6d0..bd9f4633b 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -30,7 +30,6 @@ from .. import utils from .._message import Message, MessageType from . import Sandbox, SandboxCommandError from .sandbox import _SandboxBatch -from ..storage._filebaseddirectory import FileBasedDirectory from ..storage._casbaseddirectory import CasBasedDirectory from .. import _signals from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc @@ -306,10 +305,6 @@ class SandboxRemote(Sandbox): # set up virtual dircetory upload_vdir = self.get_virtual_directory() cascache = self._get_context().get_cascache() - if isinstance(upload_vdir, FileBasedDirectory): - # Make a new temporary directory to put source in - upload_vdir = CasBasedDirectory(cascache) - upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory()) # Create directories for all marked directories. This emulates # some of the behaviour of other sandboxes, which create these -- cgit v1.2.1 From 213310e3e1890859ba5c93b15c8bdbfcafb86cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Sat, 2 Mar 2019 11:29:39 +0100 Subject: _sandboxremote.py: Request the whole directory tree as output process_job_output() currently expects the whole directory tree as output. Match this in the request by asking for the whole directory tree. --- buildstream/sandbox/_sandboxremote.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) (limited to 'buildstream/sandbox/_sandboxremote.py') diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index bd9f4633b..9e58b095a 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -283,17 +283,6 @@ class SandboxRemote(Sandbox): if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes: raise SandboxError("Output directory structure pulling from remote failed.") - path_components = os.path.split(self._output_directory) - - # Now what we have is a digest for the output. Once we return, the calling process will - # attempt to descend into our directory and find that directory, so we need to overwrite - # that. - - if not path_components: - # The artifact wants the whole directory; we could just return the returned hash in its - # place, but we don't have a means to do that yet. - raise SandboxError("Unimplemented: Output directory is empty or equal to the sandbox root.") - # At the moment, we will get the whole directory back in the first directory argument and we need # to replace the sandbox's virtual directory with that. Creating a new virtual directory object # from another hash will be interesting, though... @@ -413,11 +402,15 @@ class SandboxRemote(Sandbox): environment_variables = [remote_execution_pb2.Command. EnvironmentVariable(name=k, value=v) for (k, v) in environment.items()] + + # Request the whole directory tree as output + output_directory = os.path.relpath(os.path.sep, start=working_directory) + return remote_execution_pb2.Command(arguments=command, working_directory=working_directory, environment_variables=environment_variables, output_files=[], - output_directories=[self._output_directory], + output_directories=[output_directory], platform=None) @staticmethod -- cgit v1.2.1 From 0f124fc2fb8f1b22f892f2e2f3a9703775a7aaa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Sat, 2 Mar 2019 12:07:10 +0100 Subject: _sandboxremote.py: Process job output for failed commands The sandbox directory should be updated even for commands with non-zero exit code. This allows caching the failed buildtree for debugging. --- buildstream/sandbox/_sandboxremote.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'buildstream/sandbox/_sandboxremote.py') diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 9e58b095a..d49f31133 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -354,6 +354,9 @@ class SandboxRemote(Sandbox): operation = self.run_remote_command(channel, action_digest) action_result = self._extract_action_result(operation) + # Get output of build + self.process_job_output(action_result.output_directories, action_result.output_files) + if action_result.exit_code != 0: # A normal error during the build: the remote execution system # has worked correctly but the command failed. @@ -361,9 +364,6 @@ class SandboxRemote(Sandbox): # build command outputs which we ignore at the moment. return action_result.exit_code - # Get output of build - self.process_job_output(action_result.output_directories, action_result.output_files) - return 0 def _check_action_cache(self, action_digest): -- cgit v1.2.1 From 8c6666391abf74a1c9aaf29b21ed1c2f63739e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 3 Oct 2018 07:07:22 +0200 Subject: _sandboxremote.py: Add support for embedded stdout and stderr Part of #797. --- buildstream/sandbox/_sandboxremote.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'buildstream/sandbox/_sandboxremote.py') diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index d49f31133..be3234796 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -291,6 +291,8 @@ class SandboxRemote(Sandbox): self._set_virtual_directory(new_dir) def _run(self, command, flags, *, cwd, env): + stdout, stderr = self._get_output() + # set up virtual dircetory upload_vdir = self.get_virtual_directory() cascache = self._get_context().get_cascache() @@ -357,11 +359,16 @@ class SandboxRemote(Sandbox): # Get output of build self.process_job_output(action_result.output_directories, action_result.output_files) + if stdout: + if action_result.stdout_raw: + stdout.write(str(action_result.stdout_raw, 'utf-8', errors='ignore')) + if stderr: + if action_result.stderr_raw: + stderr.write(str(action_result.stderr_raw, 'utf-8', errors='ignore')) + if action_result.exit_code != 0: # A normal error during the build: the remote execution system # has worked correctly but the command failed. - # action_result.stdout and action_result.stderr also contains - # build command outputs which we ignore at the moment. return action_result.exit_code return 0 -- cgit v1.2.1