diff options
Diffstat (limited to 'src/buildstream')
| -rw-r--r-- | src/buildstream/data/projectconfig.yaml | 2 | ||||
| -rw-r--r-- | src/buildstream/sandbox/_sandboxbuildboxrun.py | 1 | ||||
| -rw-r--r-- | src/buildstream/sandbox/_sandboxreapi.py | 76 | ||||
| -rw-r--r-- | src/buildstream/sandbox/_sandboxremote.py | 28 | ||||
| -rw-r--r-- | src/buildstream/testing/runcli.py | 22 |
5 files changed, 71 insertions, 58 deletions
diff --git a/src/buildstream/data/projectconfig.yaml b/src/buildstream/data/projectconfig.yaml index dce4cc6aa..4b129ab75 100644 --- a/src/buildstream/data/projectconfig.yaml +++ b/src/buildstream/data/projectconfig.yaml @@ -72,7 +72,7 @@ environment-nocache: [] # Configuration for the sandbox other than environment variables # should go in 'sandbox'. Custom platform properties for Remote -# Execution commands can be set via 'remote-execution' config. +# Execution services can be set via 'remote-execution' config. sandbox: {} # Defaults for the 'split-rules' public data found on elements diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py index 3d71b7440..594a45fef 100644 --- a/src/buildstream/sandbox/_sandboxbuildboxrun.py +++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py @@ -70,6 +70,7 @@ class SandboxBuildBoxRun(SandboxREAPI): # limit support to native building on the host ISA. cls._isas.add(Platform.get_host_arch()) + # Only called when lauching a local sandbox, as we can't pre-empt the remote environment capabilities @classmethod def check_sandbox_config(cls, platform, config): if config.build_os not in cls._osfamilies: diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py index fd63ec9cb..1785db662 100644 --- a/src/buildstream/sandbox/_sandboxreapi.py +++ b/src/buildstream/sandbox/_sandboxreapi.py @@ -119,36 +119,49 @@ class SandboxREAPI(Sandbox): # Request read-write directories as output output_directories = [os.path.relpath(dir, start=working_directory) for dir in read_write_directories] - # Get the custom platform properties, if specified. These are not filtered - platform_dict = self._get_custom_platform_properties() - - # Unless explicitly disabled, generate default platform properties - if self._use_default_platform_properties(): - config = self._get_config() - default_dict = {} - default_dict["OSFamily"] = config.build_os - default_dict["ISA"] = config.build_arch - - if flags & SandboxFlags.INHERIT_UID: - uid = os.geteuid() - gid = os.getegid() + # Get the SandoxConfig + config = self._get_config() + default_dict = {} + default_dict["OSFamily"] = config.build_os + default_dict["ISA"] = config.build_arch + + if flags & SandboxFlags.INHERIT_UID: + uid = os.geteuid() + gid = os.getegid() + else: + uid = config.build_uid + gid = config.build_gid + if uid is not None: + default_dict["unixUID"] = str(uid) + if gid is not None: + default_dict["unixGID"] = str(gid) + + if flags & SandboxFlags.NETWORK_ENABLED: + default_dict["network"] = "on" + # Remove unsupported platform properties from the default dict, this filter is derived from the + # local sandbox capabilities + supported_properties = self._supported_platform_properties() + platform_dict = {key: value for (key, value) in default_dict.items() if key in supported_properties} + + # Get the platform properties dict, if specified. These are not filtered as they are specific + # to the remote server + platform_properties = self._get_platform_properties() + + # Apply the properties to the default_dict. k:v pairs in the default_dict + # can be disabled if given a explicit value of `[]` in platform properties + # with a matching key. + for platform_property, value in platform_properties.items(): + if platform_property in platform_dict: + if value != []: + raise SandboxError( + "Platform Property {}:{} should be configured in sandbox config, not remote-execution.".format( + platform_property, value + ), + reason="invalid-platform-property", + ) + del platform_dict[platform_property] else: - uid = config.build_uid - gid = config.build_gid - if uid is not None: - default_dict["unixUID"] = str(uid) - if gid is not None: - default_dict["unixGID"] = str(gid) - - if flags & SandboxFlags.NETWORK_ENABLED: - default_dict["network"] = "on" - # Remove unsupported platform properties from the default dict - supported_properties = self._supported_platform_properties() - default_dict = {key: value for (key, value) in default_dict.items() if key in supported_properties} - - # Apply the defaults to the platform_dict. Default values take precedence - # on key collisions - platform_dict = {**platform_dict, **default_dict} + platform_dict[platform_property] = value # Create Platform message with properties sorted by name in code point order platform = remote_execution_pb2.Platform() @@ -208,12 +221,9 @@ class SandboxREAPI(Sandbox): def _supported_platform_properties(self): return {"OSFamily", "ISA"} - def _get_custom_platform_properties(self): + def _get_platform_properties(self): return {} - def _use_default_platform_properties(self): - return True - # _SandboxREAPIBatch() # diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index 4439dc42c..a882a19cd 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -41,7 +41,7 @@ from .._remote import RemoteSpec class RemoteExecutionSpec( - namedtuple("RemoteExecutionSpec", "exec_service storage_service action_service custom_properties use_defaults") + namedtuple("RemoteExecutionSpec", "exec_service storage_service action_service platform_properties") ): pass @@ -98,8 +98,7 @@ class SandboxRemote(SandboxREAPI): self.action_instance = None self.action_credentials = None - self.custom_properties = config.custom_properties - self.use_defaults = config.use_defaults + self.platform_properties = config.platform_properties self.exec_instance = config.exec_service.get("instance-name", None) self.storage_instance = config.storage_service.get("instance-name", None) @@ -136,15 +135,12 @@ class SandboxRemote(SandboxREAPI): service_keys = ["execution-service", "storage-service", "action-cache-service"] - remote_config.validate_keys( - ["url", "custom-platform-properties", "default-platform-properties", *service_keys] - ) + remote_config.validate_keys(["url", "platform-properties", *service_keys]) exec_config = require_node(remote_config, "execution-service") storage_config = require_node(remote_config, "storage-service") action_config = remote_config.get_mapping("action-cache-service", default={}) - custom_properties = remote_config.get_mapping("custom-platform-properties", default={}) - use_defaults = remote_config.get_bool("default-platform-properties", default=True) + platform_properties = remote_config.get_mapping("platform-properties", default={}) tls_keys = ["client-key", "client-cert", "server-cert"] @@ -190,11 +186,11 @@ class SandboxRemote(SandboxREAPI): if tls_key in config: config[tls_key] = resolve_path(config.get_str(tls_key)) - # Add in the custom platform properties config - service_configs.append(custom_properties) + # Add in the platform properties config + service_configs.append(platform_properties) # TODO: we should probably not be stripping node info and rather load files the safe way - return RemoteExecutionSpec(*[conf.strip_node_info() for conf in service_configs], use_defaults) + return RemoteExecutionSpec(*[conf.strip_node_info() for conf in service_configs]) def run_remote_command(self, channel, action_digest): # Sends an execution request to the remote execution server. @@ -442,13 +438,9 @@ class SandboxRemote(SandboxREAPI): self.info("Action result found in action cache") return result - def _get_custom_platform_properties(self): - # Dict containing custom platformn properties, if provided in config - return self.custom_properties - - def _use_default_platform_properties(self): - # Bool, defaults to True unless overriden in RE Spec - return self.use_defaults + def _get_platform_properties(self): + # Dict platformn properties, if provided in config + return self.platform_properties @staticmethod def _extract_action_result(operation): diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py index be72f6ae0..689a5c69c 100644 --- a/src/buildstream/testing/runcli.py +++ b/src/buildstream/testing/runcli.py @@ -805,14 +805,24 @@ def cli_remote_execution(tmpdir, remote_services): remote_execution["storage-service"] = { "url": remote_services.storage_service, } - if remote_services.custom_properties: - # Expected string with substring pattern "--platform property=value" - remote_execution["custom-platform-properties"] = dict( - s.split("=") for s in remote_services.custom_properties.replace("--platform", "").split() + if remote_services.platform_properties: + default_properties = ["OSFamily", "ISA"] + # Strip cli argument, expected string with substring pattern "--platform property=value" + parsed_properties = dict( + s.split("=") for s in remote_services.platform_properties.replace("--platform", "").split() ) + # If a default property has been set on the server, do not add it to remote-execution config + # as these should configured via sandbox config in bst if required. If a default hasn't been + # set on the server (e.g, no ISO) then this maps to it needing to be explicitly disabled in bst. + for default_property in default_properties: + if default_property in parsed_properties: + del parsed_properties[default_property] + else: + parsed_properties[default_property] = [] + + remote_execution["platform-properties"] = parsed_properties + if remote_execution: - if not remote_services.use_defaults: - remote_execution["default-platform-properties"] = False fixture.configure({"remote-execution": remote_execution}) if remote_services.source_service: |