Add support for user remote execution configuration

Fixes #631.

5 files changed, 72 insertions, 4 deletions

diff --git a/buildstream/_context.py b/buildstream/_context.py
index e0eea99ff..c62755cf2 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
 from ._artifactcache.cascache import CASCache
 from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
 from .plugin import _plugin_lookup
+from .sandbox import SandboxRemote
 
 
 # Context()
@@ -72,6 +73,9 @@ class Context():
         # The locations from which to push and pull prebuilt artifacts
         self.artifact_cache_specs = None
 
+        # The global remote execution configuration
+        self.remote_execution_specs = None
+
         # The directory to store build logs
         self.logdir = None
 
@@ -187,7 +191,7 @@ class Context():
         _yaml.node_validate(defaults, [
             'sourcedir', 'builddir', 'artifactdir', 'logdir',
             'scheduler', 'artifacts', 'logging', 'projects',
-            'cache', 'prompt', 'workspacedir',
+            'cache', 'prompt', 'workspacedir', 'remote-execution'
         ])
 
         for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
@@ -212,6 +216,8 @@ class Context():
         # Load artifact share configuration
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
 
+        self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
+
         # Load pull build trees configuration
         self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
 
@@ -271,7 +277,8 @@ class Context():
         # Shallow validation of overrides, parts of buildstream which rely
         # on the overrides are expected to validate elsewhere.
         for _, overrides in _yaml.node_items(self._project_overrides):
-            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
+            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
+                                            'remote-execution'])
 
         profile_end(Topics.LOAD_CONTEXT, 'load')
 
diff --git a/buildstream/_project.py b/buildstream/_project.py
index ef8d835cf..1492fde77 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -507,7 +507,16 @@ class Project():
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
 
         # Load remote-execution configuration for this project
-        self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+        project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+        override_specs = SandboxRemote.specs_from_config_node(
+            self._context.get_overrides(self.name), self.directory)
+
+        if override_specs is not None:
+            self.remote_execution_specs = override_specs
+        elif project_specs is not None:
+            self.remote_execution_specs = project_specs
+        else:
+            self.remote_execution_specs = self._context.remote_execution_specs
 
         # Load sandbox environment variables
         self.base_environment = _yaml.node_get(config, Mapping, 'environment')
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 6fa51e3e6..9b54f2e58 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -81,7 +81,7 @@ class SandboxRemote(Sandbox):
         self._get_context().message(Message(None, MessageType.INFO, msg))
 
     @staticmethod
-    def specs_from_config_node(config_node, basedir):
+    def specs_from_config_node(config_node, basedir=None):
 
         def require_node(config, keyname):
             val = config.get(keyname)
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index bb66231cb..08e8a0861 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
 push/pull access. For information regarding the server/client certificates
 and keys, please see: :ref:`Key pair for the server <server_authentication>`.
 
+.. _project_remote_execution:
 
 Remote execution
 ~~~~~~~~~~~~~~~~
@@ -268,6 +269,9 @@ instance names.
 
 The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
 
+Remote execution configuration can be also provided in the `user
+configuration <user_config_remote_execution>`.
+
 .. _project_essentials_mirrors:
 
 Mirrors
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 67574e5c4..71ca64a4a 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
 server/client certificates and keys, please see:
 :ref:`Key pair for the server <server_authentication>`.
 
+.. _user_config_remote_execution:
+
+Remote execution
+~~~~~~~~~~~~~~~~
+
+The same configuration for :ref:`remote execution <project_remote_execution>`
+in ``project.conf`` can be provided in the user configuation.
+
+There is only one remote execution configuration used per project.
+
+The project overrides will be taken in priority. The global
+configuration will be used as fallback.
+
+1. Global remote execution fallback:
+
+.. code:: yaml
+
+  remote-execution:
+    execution-service:
+      url: http://execution.fallback.example.com:50051
+      instance-name: main
+    storage-service:
+      url: https://storage.fallback.example.com:11002/
+      server-cert: /keys/server.crt
+      client-cert: /keys/client.crt
+      client-key: /keys/client.key
+      instance-name: main
+    action-cache-service:
+      url: http://action.flalback.example.com:50052
+
+2. Project override:
+
+.. code:: yaml
+
+  projects:
+    some_project:
+      remote-execution:
+        execution-service:
+          url: http://execution.some_project.example.com:50051
+          instance-name: main
+        storage-service:
+          url: https://storage.some_project.example.com:11002/
+          server-cert: /some_project_keys/server.crt
+          client-cert: /some_project_keys/client.crt
+          client-key: /some_project_keys/client.key
+          instance-name: main
+        action-cache-service:
+          url: http://action.some_project.example.com:50052
 
 
 Strict build plan