diff options
author | Valentin David <valentin.david@codethink.co.uk> | 2018-12-20 12:59:43 +0100 |
---|---|---|
committer | Valentin David <valentin.david@gmail.com> | 2019-01-09 10:45:23 +0000 |
commit | 605f8d11dc830b7d3243735d1f340e16364c8d8c (patch) | |
tree | 19d7afbfae20cd5fb332cbc484d6dd0c164c416f | |
parent | 17c5ca2d48fff08df711c76954cf3d92a1b10550 (diff) | |
download | buildstream-605f8d11dc830b7d3243735d1f340e16364c8d8c.tar.gz |
Add support for user remote execution configuration
Fixes #631.
-rw-r--r-- | buildstream/_context.py | 11 | ||||
-rw-r--r-- | buildstream/_project.py | 11 | ||||
-rw-r--r-- | buildstream/sandbox/_sandboxremote.py | 2 | ||||
-rw-r--r-- | doc/source/format_project.rst | 4 | ||||
-rw-r--r-- | doc/source/using_config.rst | 48 |
5 files changed, 72 insertions, 4 deletions
diff --git a/buildstream/_context.py b/buildstream/_context.py index e0eea99ff..c62755cf2 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache from ._artifactcache.cascache import CASCache from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE from .plugin import _plugin_lookup +from .sandbox import SandboxRemote # Context() @@ -72,6 +73,9 @@ class Context(): # The locations from which to push and pull prebuilt artifacts self.artifact_cache_specs = None + # The global remote execution configuration + self.remote_execution_specs = None + # The directory to store build logs self.logdir = None @@ -187,7 +191,7 @@ class Context(): _yaml.node_validate(defaults, [ 'sourcedir', 'builddir', 'artifactdir', 'logdir', 'scheduler', 'artifacts', 'logging', 'projects', - 'cache', 'prompt', 'workspacedir', + 'cache', 'prompt', 'workspacedir', 'remote-execution' ]) for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']: @@ -212,6 +216,8 @@ class Context(): # Load artifact share configuration self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults) + self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults) + # Load pull build trees configuration self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees') @@ -271,7 +277,8 @@ class Context(): # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. for _, overrides in _yaml.node_items(self._project_overrides): - _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror']) + _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror', + 'remote-execution']) profile_end(Topics.LOAD_CONTEXT, 'load') diff --git a/buildstream/_project.py b/buildstream/_project.py index ef8d835cf..1492fde77 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -507,7 +507,16 @@ class Project(): self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory) # Load remote-execution configuration for this project - self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory) + project_specs = SandboxRemote.specs_from_config_node(config, self.directory) + override_specs = SandboxRemote.specs_from_config_node( + self._context.get_overrides(self.name), self.directory) + + if override_specs is not None: + self.remote_execution_specs = override_specs + elif project_specs is not None: + self.remote_execution_specs = project_specs + else: + self.remote_execution_specs = self._context.remote_execution_specs # Load sandbox environment variables self.base_environment = _yaml.node_get(config, Mapping, 'environment') diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 6fa51e3e6..9b54f2e58 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -81,7 +81,7 @@ class SandboxRemote(Sandbox): self._get_context().message(Message(None, MessageType.INFO, msg)) @staticmethod - def specs_from_config_node(config_node, basedir): + def specs_from_config_node(config_node, basedir=None): def require_node(config, keyname): val = config.get(keyname) diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst index bb66231cb..08e8a0861 100644 --- a/doc/source/format_project.rst +++ b/doc/source/format_project.rst @@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and push/pull access. For information regarding the server/client certificates and keys, please see: :ref:`Key pair for the server <server_authentication>`. +.. _project_remote_execution: Remote execution ~~~~~~~~~~~~~~~~ @@ -268,6 +269,9 @@ instance names. The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis. +Remote execution configuration can be also provided in the `user +configuration <user_config_remote_execution>`. + .. _project_essentials_mirrors: Mirrors diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index 67574e5c4..71ca64a4a 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the server/client certificates and keys, please see: :ref:`Key pair for the server <server_authentication>`. +.. _user_config_remote_execution: + +Remote execution +~~~~~~~~~~~~~~~~ + +The same configuration for :ref:`remote execution <project_remote_execution>` +in ``project.conf`` can be provided in the user configuation. + +There is only one remote execution configuration used per project. + +The project overrides will be taken in priority. The global +configuration will be used as fallback. + +1. Global remote execution fallback: + +.. code:: yaml + + remote-execution: + execution-service: + url: http://execution.fallback.example.com:50051 + instance-name: main + storage-service: + url: https://storage.fallback.example.com:11002/ + server-cert: /keys/server.crt + client-cert: /keys/client.crt + client-key: /keys/client.key + instance-name: main + action-cache-service: + url: http://action.flalback.example.com:50052 + +2. Project override: + +.. code:: yaml + + projects: + some_project: + remote-execution: + execution-service: + url: http://execution.some_project.example.com:50051 + instance-name: main + storage-service: + url: https://storage.some_project.example.com:11002/ + server-cert: /some_project_keys/server.crt + client-cert: /some_project_keys/client.crt + client-key: /some_project_keys/client.key + instance-name: main + action-cache-service: + url: http://action.some_project.example.com:50052 Strict build plan |