Merge branch 'valentindavid/remote_execution_configuration' into 'master'

Remote execution configuration: HTTPS and user configuration

Closes #780 and #631

See merge request BuildStream/buildstream!1030

5 files changed, 115 insertions, 13 deletions

diff --git a/buildstream/_context.py b/buildstream/_context.py
index e0eea99ff..c62755cf2 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
 from ._artifactcache.cascache import CASCache
 from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
 from .plugin import _plugin_lookup
+from .sandbox import SandboxRemote
 
 
 # Context()
@@ -72,6 +73,9 @@ class Context():
         # The locations from which to push and pull prebuilt artifacts
         self.artifact_cache_specs = None
 
+        # The global remote execution configuration
+        self.remote_execution_specs = None
+
         # The directory to store build logs
         self.logdir = None
 
@@ -187,7 +191,7 @@ class Context():
         _yaml.node_validate(defaults, [
             'sourcedir', 'builddir', 'artifactdir', 'logdir',
             'scheduler', 'artifacts', 'logging', 'projects',
-            'cache', 'prompt', 'workspacedir',
+            'cache', 'prompt', 'workspacedir', 'remote-execution'
         ])
 
         for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
@@ -212,6 +216,8 @@ class Context():
         # Load artifact share configuration
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
 
+        self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
+
         # Load pull build trees configuration
         self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
 
@@ -271,7 +277,8 @@ class Context():
         # Shallow validation of overrides, parts of buildstream which rely
         # on the overrides are expected to validate elsewhere.
         for _, overrides in _yaml.node_items(self._project_overrides):
-            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
+            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
+                                            'remote-execution'])
 
         profile_end(Topics.LOAD_CONTEXT, 'load')
 
diff --git a/buildstream/_project.py b/buildstream/_project.py
index ef8d835cf..1492fde77 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -507,7 +507,16 @@ class Project():
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
 
         # Load remote-execution configuration for this project
-        self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+        project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+        override_specs = SandboxRemote.specs_from_config_node(
+            self._context.get_overrides(self.name), self.directory)
+
+        if override_specs is not None:
+            self.remote_execution_specs = override_specs
+        elif project_specs is not None:
+            self.remote_execution_specs = project_specs
+        else:
+            self.remote_execution_specs = self._context.remote_execution_specs
 
         # Load sandbox environment variables
         self.base_environment = _yaml.node_get(config, Mapping, 'environment')
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 503cf931a..a842f08d3 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -62,10 +62,32 @@ class SandboxRemote(Sandbox):
         self.storage_url = config.storage_service['url']
         self.exec_url = config.exec_service['url']
 
+        exec_certs = {}
+        for key in ['client-cert', 'client-key', 'server-cert']:
+            if key in config.exec_service:
+                with open(config.exec_service[key], 'rb') as f:
+                    exec_certs[key] = f.read()
+
+        self.exec_credentials = grpc.ssl_channel_credentials(
+            root_certificates=exec_certs.get('server-cert'),
+            private_key=exec_certs.get('client-key'),
+            certificate_chain=exec_certs.get('client-cert'))
+
+        action_certs = {}
+        for key in ['client-cert', 'client-key', 'server-cert']:
+            if key in config.action_service:
+                with open(config.action_service[key], 'rb') as f:
+                    action_certs[key] = f.read()
+
         if config.action_service:
             self.action_url = config.action_service['url']
+            self.action_credentials = grpc.ssl_channel_credentials(
+                root_certificates=action_certs.get('server-cert'),
+                private_key=action_certs.get('client-key'),
+                certificate_chain=action_certs.get('client-cert'))
         else:
             self.action_url = None
+            self.action_credentials = None
 
         self.server_instance = config.exec_service.get('instance', None)
         self.storage_instance = config.storage_service.get('instance', None)
@@ -81,7 +103,7 @@ class SandboxRemote(Sandbox):
         self._get_context().message(Message(None, MessageType.INFO, msg))
 
     @staticmethod
-    def specs_from_config_node(config_node, basedir):
+    def specs_from_config_node(config_node, basedir=None):
 
         def require_node(config, keyname):
             val = config.get(keyname)
@@ -109,10 +131,10 @@ class SandboxRemote(Sandbox):
         remote_exec_storage_config = require_node(remote_config, 'storage-service')
         remote_exec_action_config = remote_config.get('action-cache-service', {})
 
-        _yaml.node_validate(remote_exec_service_config, ['url', 'instance'])
+        _yaml.node_validate(remote_exec_service_config, ['url', 'instance'] + tls_keys)
         _yaml.node_validate(remote_exec_storage_config, ['url', 'instance'] + tls_keys)
         if remote_exec_action_config:
-            _yaml.node_validate(remote_exec_action_config, ['url'])
+            _yaml.node_validate(remote_exec_action_config, ['url'] + tls_keys)
         else:
             remote_config['action-service'] = None
 
@@ -135,6 +157,19 @@ class SandboxRemote(Sandbox):
                                       "remote-execution configuration. Your config is missing '{}'."
                                       .format(str(provenance), tls_keys, key))
 
+        def resolve_path(path):
+            if basedir and path:
+                return os.path.join(basedir, path)
+            else:
+                return path
+
+        for key in tls_keys:
+            for d in (remote_config['execution-service'],
+                      remote_config['storage-service'],
+                      remote_exec_action_config):
+                if key in d:
+                    d[key] = resolve_path(d[key])
+
         spec = RemoteExecutionSpec(remote_config['execution-service'],
                                    remote_config['storage-service'],
                                    remote_exec_action_config)
@@ -295,6 +330,8 @@ class SandboxRemote(Sandbox):
                                "for example: http://buildservice:50051.")
         if url.scheme == 'http':
             channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
+        elif url.scheme == 'https':
+            channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials)
         else:
             raise SandboxError("Remote execution currently only supports the 'http' protocol "
                                "and '{}' was supplied.".format(url.scheme))
@@ -352,11 +389,11 @@ class SandboxRemote(Sandbox):
         if not url.port:
             raise SandboxError("You must supply a protocol and port number in the action-cache-service url, "
                                "for example: http://buildservice:50051.")
-        if not url.scheme == "http":
-            raise SandboxError("Currently only support http for the action cache"
-                               "and {} was supplied".format(url.scheme))
+        if url.scheme == 'http':
+            channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
+        elif url.scheme == 'https':
+            channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials)
 
-        channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
         request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
         stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
         try:
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index bb66231cb..c3555e0c1 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
 push/pull access. For information regarding the server/client certificates
 and keys, please see: :ref:`Key pair for the server <server_authentication>`.
 
+.. _project_remote_execution:
 
 Remote execution
 ~~~~~~~~~~~~~~~~
@@ -243,9 +244,6 @@ using the `remote-execution` option:
     action-cache-service:
       url: http://bar.action.com:50052
 
-The execution-service part of remote execution does not support encrypted
-connections yet, so the protocol must always be http.
-
 storage-service specifies a remote CAS store and the parameters are the
 same as those used to specify an :ref:`artifact server <artifacts>`.
 
@@ -268,6 +266,9 @@ instance names.
 
 The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
 
+Remote execution configuration can be also provided in the `user
+configuration <user_config_remote_execution>`.
+
 .. _project_essentials_mirrors:
 
 Mirrors
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 67574e5c4..71ca64a4a 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
 server/client certificates and keys, please see:
 :ref:`Key pair for the server <server_authentication>`.
 
+.. _user_config_remote_execution:
+
+Remote execution
+~~~~~~~~~~~~~~~~
+
+The same configuration for :ref:`remote execution <project_remote_execution>`
+in ``project.conf`` can be provided in the user configuation.
+
+There is only one remote execution configuration used per project.
+
+The project overrides will be taken in priority. The global
+configuration will be used as fallback.
+
+1. Global remote execution fallback:
+
+.. code:: yaml
+
+  remote-execution:
+    execution-service:
+      url: http://execution.fallback.example.com:50051
+      instance-name: main
+    storage-service:
+      url: https://storage.fallback.example.com:11002/
+      server-cert: /keys/server.crt
+      client-cert: /keys/client.crt
+      client-key: /keys/client.key
+      instance-name: main
+    action-cache-service:
+      url: http://action.flalback.example.com:50052
+
+2. Project override:
+
+.. code:: yaml
+
+  projects:
+    some_project:
+      remote-execution:
+        execution-service:
+          url: http://execution.some_project.example.com:50051
+          instance-name: main
+        storage-service:
+          url: https://storage.some_project.example.com:11002/
+          server-cert: /some_project_keys/server.crt
+          client-cert: /some_project_keys/client.crt
+          client-key: /some_project_keys/client.key
+          instance-name: main
+        action-cache-service:
+          url: http://action.some_project.example.com:50052
 
 
 Strict build plan