diff options
| -rw-r--r-- | buildstream/_cas/casremote.py | 35 | ||||
| -rw-r--r-- | buildstream/sandbox/_sandboxremote.py | 17 | ||||
| -rw-r--r-- | doc/source/format_project.rst | 1 | ||||
| -rw-r--r-- | doc/source/using_config.rst | 10 |
4 files changed, 47 insertions, 16 deletions
diff --git a/buildstream/_cas/casremote.py b/buildstream/_cas/casremote.py index 56ba4c5d8..df1dd799c 100644 --- a/buildstream/_cas/casremote.py +++ b/buildstream/_cas/casremote.py @@ -31,7 +31,7 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key # @staticmethod def _new_from_config_node(spec_node, basedir=None): - _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance_name']) + _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance-name']) url = _yaml.node_get(spec_node, str, 'url') push = _yaml.node_get(spec_node, bool, 'push', default_value=False) if not url: @@ -39,7 +39,7 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key raise LoadError(LoadErrorReason.INVALID_DATA, "{}: empty artifact cache URL".format(provenance)) - instance_name = _yaml.node_get(spec_node, str, 'server-cert', default_value=None) + instance_name = _yaml.node_get(spec_node, str, 'instance-name', default_value=None) server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None) if server_cert and basedir: @@ -83,6 +83,7 @@ class CASRemote(): self.spec = spec self._initialized = False self.channel = None + self.instance_name = None self.bytestream = None self.cas = None self.ref_storage = None @@ -125,6 +126,8 @@ class CASRemote(): else: raise CASRemoteError("Unsupported URL: {}".format(self.spec.url)) + self.instance_name = self.spec.instance_name or None + self.bytestream = bytestream_pb2_grpc.ByteStreamStub(self.channel) self.cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self.channel) self.capabilities = remote_execution_pb2_grpc.CapabilitiesStub(self.channel) @@ -133,6 +136,8 @@ class CASRemote(): self.max_batch_total_size_bytes = _MAX_PAYLOAD_BYTES try: request = remote_execution_pb2.GetCapabilitiesRequest() + if self.instance_name: + request.instance_name = self.instance_name response = self.capabilities.GetCapabilities(request) server_max_batch_total_size_bytes = response.cache_capabilities.max_batch_total_size_bytes if 0 < server_max_batch_total_size_bytes < self.max_batch_total_size_bytes: @@ -146,6 +151,8 @@ class CASRemote(): self.batch_read_supported = False try: request = remote_execution_pb2.BatchReadBlobsRequest() + if self.instance_name: + request.instance_name = self.instance_name response = self.cas.BatchReadBlobs(request) self.batch_read_supported = True except grpc.RpcError as e: @@ -156,6 +163,8 @@ class CASRemote(): self.batch_update_supported = False try: request = remote_execution_pb2.BatchUpdateBlobsRequest() + if self.instance_name: + request.instance_name = self.instance_name response = self.cas.BatchUpdateBlobs(request) self.batch_update_supported = True except grpc.RpcError as e: @@ -224,6 +233,8 @@ class CASRemote(): self.init() request = remote_execution_pb2.FindMissingBlobsRequest() + if self.instance_name: + request.instance_name = self.instance_name request.blob_digests.extend([digest]) response = self.cas.FindMissingBlobs(request) @@ -258,7 +269,13 @@ class CASRemote(): # Local Private Methods # ################################################ def _fetch_blob(self, digest, stream): - resource_name = '/'.join(['blobs', digest.hash, str(digest.size_bytes)]) + if self.instance_name: + resource_name = '/'.join([self.instance_name, 'blobs', + digest.hash, str(digest.size_bytes)]) + else: + resource_name = '/'.join(['blobs', + digest.hash, str(digest.size_bytes)]) + request = bytestream_pb2.ReadRequest() request.resource_name = resource_name request.read_offset = 0 @@ -269,8 +286,12 @@ class CASRemote(): assert digest.size_bytes == os.fstat(stream.fileno()).st_size def _send_blob(self, digest, stream, u_uid=uuid.uuid4()): - resource_name = '/'.join(['uploads', str(u_uid), 'blobs', - digest.hash, str(digest.size_bytes)]) + if self.instance_name: + resource_name = '/'.join([self.instance_name, 'uploads', str(u_uid), 'blobs', + digest.hash, str(digest.size_bytes)]) + else: + resource_name = '/'.join(['uploads', str(u_uid), 'blobs', + digest.hash, str(digest.size_bytes)]) def request_stream(resname, instream): offset = 0 @@ -304,6 +325,8 @@ class _CASBatchRead(): self._remote = remote self._max_total_size_bytes = remote.max_batch_total_size_bytes self._request = remote_execution_pb2.BatchReadBlobsRequest() + if remote.instance_name: + self._request.instance_name = remote.instance_name self._size = 0 self._sent = False @@ -351,6 +374,8 @@ class _CASBatchUpdate(): self._remote = remote self._max_total_size_bytes = remote.max_batch_total_size_bytes self._request = remote_execution_pb2.BatchUpdateBlobsRequest() + if remote.instance_name: + self._request.instance_name = remote.instance_name self._size = 0 self._sent = False diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py index 6a1f6f2f8..e9a4a7cb5 100644 --- a/buildstream/sandbox/_sandboxremote.py +++ b/buildstream/sandbox/_sandboxremote.py @@ -81,16 +81,18 @@ class SandboxRemote(Sandbox): if config.action_service: self.action_url = config.action_service['url'] + self.action_instance = config.action_service.get('instance-name', None) self.action_credentials = grpc.ssl_channel_credentials( root_certificates=action_certs.get('server-cert'), private_key=action_certs.get('client-key'), certificate_chain=action_certs.get('client-cert')) else: self.action_url = None + self.action_instance = None self.action_credentials = None - self.server_instance = config.exec_service.get('instance', None) - self.storage_instance = config.storage_service.get('instance', None) + self.exec_instance = config.exec_service.get('instance-name', None) + self.storage_instance = config.storage_service.get('instance-name', None) self.storage_remote_spec = CASRemoteSpec(self.storage_url, push=True, server_cert=config.storage_service['server-cert'], @@ -131,10 +133,10 @@ class SandboxRemote(Sandbox): remote_exec_storage_config = require_node(remote_config, 'storage-service') remote_exec_action_config = remote_config.get('action-cache-service', {}) - _yaml.node_validate(remote_exec_service_config, ['url', 'instance'] + tls_keys) - _yaml.node_validate(remote_exec_storage_config, ['url', 'instance'] + tls_keys) + _yaml.node_validate(remote_exec_service_config, ['url', 'instance-name'] + tls_keys) + _yaml.node_validate(remote_exec_storage_config, ['url', 'instance-name'] + tls_keys) if remote_exec_action_config: - _yaml.node_validate(remote_exec_action_config, ['url'] + tls_keys) + _yaml.node_validate(remote_exec_action_config, ['url', 'instance-name'] + tls_keys) else: remote_config['action-service'] = None @@ -182,7 +184,7 @@ class SandboxRemote(Sandbox): # Try to create a communication channel to the BuildGrid server. stub = remote_execution_pb2_grpc.ExecutionStub(channel) - request = remote_execution_pb2.ExecuteRequest(instance_name=self.server_instance, + request = remote_execution_pb2.ExecuteRequest(instance_name=self.exec_instance, action_digest=action_digest, skip_cache_lookup=False) @@ -394,7 +396,8 @@ class SandboxRemote(Sandbox): elif url.scheme == 'https': channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials) - request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest) + request = remote_execution_pb2.GetActionResultRequest(instance_name=self.action_instance, + action_digest=action_digest) stub = remote_execution_pb2_grpc.ActionCacheStub(channel) try: result = stub.GetActionResult(request) diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst index f4dea333a..529816176 100644 --- a/doc/source/format_project.rst +++ b/doc/source/format_project.rst @@ -243,6 +243,7 @@ using the `remote-execution` option: instance-name: development-emea-1 action-cache-service: url: http://bar.action.com:50052 + instance-name: development-emea-1 storage-service specifies a remote CAS store and the parameters are the same as those used to specify an :ref:`artifact server <artifacts>`. diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index 71ca64a4a..a7ee4dca9 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -122,13 +122,14 @@ configuration will be used as fallback. url: http://execution.fallback.example.com:50051 instance-name: main storage-service: - url: https://storage.fallback.example.com:11002/ + url: https://storage.fallback.example.com:11002 server-cert: /keys/server.crt client-cert: /keys/client.crt client-key: /keys/client.key instance-name: main action-cache-service: - url: http://action.flalback.example.com:50052 + url: http://cache.flalback.example.com:50052 + instance-name: main 2. Project override: @@ -141,13 +142,14 @@ configuration will be used as fallback. url: http://execution.some_project.example.com:50051 instance-name: main storage-service: - url: https://storage.some_project.example.com:11002/ + url: https://storage.some_project.example.com:11002 server-cert: /some_project_keys/server.crt client-cert: /some_project_keys/client.crt client-key: /some_project_keys/client.key instance-name: main action-cache-service: - url: http://action.some_project.example.com:50052 + url: http://cache.some_project.example.com:50052 + instance-name: main Strict build plan |