diff options
-rw-r--r-- | buildstream/_cas/casserver.py | 83 | ||||
-rw-r--r-- | buildstream/_protos/buildstream/v2/artifact.proto | 88 | ||||
-rw-r--r-- | buildstream/_protos/buildstream/v2/artifact_pb2.py | 387 | ||||
-rw-r--r-- | buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py | 68 | ||||
-rw-r--r-- | tests/artifactcache/artifactservice.py | 109 |
5 files changed, 734 insertions, 1 deletions
diff --git a/buildstream/_cas/casserver.py b/buildstream/_cas/casserver.py index 8f8ef4efe..f88db717a 100644 --- a/buildstream/_cas/casserver.py +++ b/buildstream/_cas/casserver.py @@ -28,12 +28,14 @@ import errno import threading import grpc +from google.protobuf.message import DecodeError import click from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc -from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc from .._protos.google.rpc import code_pb2 +from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \ + artifact_pb2, artifact_pb2_grpc from .._exceptions import CASError @@ -62,6 +64,7 @@ def create_server(repo, *, enable_push, max_head_size=int(10e9), min_head_size=int(2e9)): cas = CASCache(os.path.abspath(repo)) + artifactdir = os.path.join(os.path.abspath(repo), 'artifacts', 'refs') # Use max_workers default from Python 3.5+ max_workers = (os.cpu_count() or 1) * 5 @@ -81,6 +84,9 @@ def create_server(repo, *, enable_push, buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server( _ReferenceStorageServicer(cas, enable_push=enable_push), server) + artifact_pb2_grpc.add_ArtifactServiceServicer_to_server( + _ArtifactServicer(cas, artifactdir), server) + return server @@ -405,6 +411,81 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer): return response +class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer): + + def __init__(self, cas, artifactdir): + super().__init__() + self.cas = cas + self.artifactdir = artifactdir + os.makedirs(artifactdir, exist_ok=True) + + def GetArtifact(self, request, context): + artifact_path = os.path.join(self.artifactdir, request.cache_key) + if not os.path.exists(artifact_path): + context.abort(grpc.StatusCode.NOT_FOUND, "Artifact proto not found") + + artifact = artifact_pb2.Artifact() + with open(artifact_path, 'rb') as f: + artifact.ParseFromString(f.read()) + + files_digest = artifact.files + + # Now update mtimes of files present. + try: + self.cas.update_tree_mtime(files_digest) + except FileNotFoundError: + os.unlink(artifact_path) + context.abort(grpc.StatusCode.NOT_FOUND, + "Artifact files incomplete") + except DecodeError: + context.abort(grpc.StatusCode.NOT_FOUND, + "Artifact files not valid") + + return artifact + + def UpdateArtifact(self, request, context): + artifact = request.artifact + + # Check that the files specified are in the CAS + self._check_directory("files", artifact.files, context) + + # Unset protocol buffers don't evaluated to False but do return empty + # strings, hence str() + if str(artifact.buildtree): + self._check_directory("buildtree", artifact.buildtree, context) + + if str(artifact.public_data): + self._check_file("public data", artifact.public_data, context) + + for log_file in artifact.logs: + self._check_file("log digest", log_file.digest, context) + + # Add the artifact proto to the cas + artifact_path = os.path.join(self.artifactdir, request.cache_key) + os.makedirs(os.path.dirname(artifact_path), exist_ok=True) + with open(artifact_path, 'wb') as f: + f.write(artifact.SerializeToString()) + + return artifact + + def _check_directory(self, name, digest, context): + try: + directory = remote_execution_pb2.Directory() + with open(self.cas.objpath(digest), 'rb') as f: + directory.ParseFromString(f.read()) + except FileNotFoundError: + context.abort(grpc.StatusCode.FAILED_PRECONDITION, + "Artifact {} specified but no files found".format(name)) + except DecodeError: + context.abort(grpc.StatusCode.FAILED_PRECONDITION, + "Artifact {} specified but directory not found".format(name)) + + def _check_file(self, name, digest, context): + if not os.path.exists(self.cas.objpath(digest)): + context.abort(grpc.StatusCode.FAILED_PRECONDITION, + "Artifact {} specified but not found".format(name)) + + def _digest_from_download_resource_name(resource_name): parts = resource_name.split('/') diff --git a/buildstream/_protos/buildstream/v2/artifact.proto b/buildstream/_protos/buildstream/v2/artifact.proto new file mode 100644 index 000000000..56ddbca6b --- /dev/null +++ b/buildstream/_protos/buildstream/v2/artifact.proto @@ -0,0 +1,88 @@ +// Copyright 2019 Bloomberg Finance LP +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Authors +// Raoul Hidalgo Charman <raoul.hidalgo.charman@gmail.com> + +syntax = "proto3"; + +package buildstream.v2; + +import "build/bazel/remote/execution/v2/remote_execution.proto"; +import "google/api/annotations.proto"; + +service ArtifactService { + // Retrieves an Artifact message + // + // Errors: + // * `NOT_FOUND`: Artifact not found on server + rpc GetArtifact(GetArtifactRequest) returns (Artifact) {} + + // Sets an Artifact message + // + // Errors: + // * `FAILED_PRECONDITION`: Files specified in upload aren't present in CAS + rpc UpdateArtifact(UpdateArtifactRequest) returns (Artifact) {} +} + +message Artifact { + // This version number must always be present and can be used to + // further indicate presence or absence of parts of the proto at a + // later date. It only needs incrementing if a change to what is + // *mandatory* changes. + int32 version = 1; + // Core metadata + bool build_success = 2; + string build_error = 3; // optional + string build_error_details = 4; + string strong_key = 5; + string weak_key = 6; + bool was_workspaced = 7; + // digest of a Directory + build.bazel.remote.execution.v2.Digest files = 8; + + // Information about the build dependencies + message Dependency { + string element_name = 1; + string cache_key = 2; + bool was_workspaced = 3; + }; + repeated Dependency build_deps = 9; + + // The public data is a yaml file which is stored into the CAS + // Digest is of a directory + build.bazel.remote.execution.v2.Digest public_data = 10; + + // The logs are stored in the CAS + message LogFile { + string name = 1; + // digest of a file + build.bazel.remote.execution.v2.Digest digest = 2; + }; + repeated LogFile logs = 11; // Zero or more log files here + + // digest of a directory + build.bazel.remote.execution.v2.Digest buildtree = 12; // optional +} + +message GetArtifactRequest { + string instance_name = 1; + string cache_key = 2; +} + +message UpdateArtifactRequest { + string instance_name = 1; + string cache_key = 2; + Artifact artifact = 3; +} diff --git a/buildstream/_protos/buildstream/v2/artifact_pb2.py b/buildstream/_protos/buildstream/v2/artifact_pb2.py new file mode 100644 index 000000000..c56d1ae8a --- /dev/null +++ b/buildstream/_protos/buildstream/v2/artifact_pb2.py @@ -0,0 +1,387 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: buildstream/v2/artifact.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 +from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='buildstream/v2/artifact.proto', + package='buildstream.v2', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\xde\x04\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x18\x06 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x07 \x01(\x08\x12\x36\n\x05\x66iles\x18\x08 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x37\n\nbuild_deps\x18\t \x03(\x0b\x32#.buildstream.v2.Artifact.Dependency\x12<\n\x0bpublic_data\x18\n \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12.\n\x04logs\x18\x0b \x03(\x0b\x32 .buildstream.v2.Artifact.LogFile\x12:\n\tbuildtree\x18\x0c \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x1aM\n\nDependency\x12\x14\n\x0c\x65lement_name\x18\x01 \x01(\t\x12\x11\n\tcache_key\x18\x02 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x03 \x01(\x08\x1aP\n\x07LogFile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\">\n\x12GetArtifactRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x11\n\tcache_key\x18\x02 \x01(\t\"m\n\x15UpdateArtifactRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x11\n\tcache_key\x18\x02 \x01(\t\x12*\n\x08\x61rtifact\x18\x03 \x01(\x0b\x32\x18.buildstream.v2.Artifact2\xb5\x01\n\x0f\x41rtifactService\x12M\n\x0bGetArtifact\x12\".buildstream.v2.GetArtifactRequest\x1a\x18.buildstream.v2.Artifact\"\x00\x12S\n\x0eUpdateArtifact\x12%.buildstream.v2.UpdateArtifactRequest\x1a\x18.buildstream.v2.Artifact\"\x00\x62\x06proto3') + , + dependencies=[build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.DESCRIPTOR,google_dot_api_dot_annotations__pb2.DESCRIPTOR,]) + + + + +_ARTIFACT_DEPENDENCY = _descriptor.Descriptor( + name='Dependency', + full_name='buildstream.v2.Artifact.Dependency', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='element_name', full_name='buildstream.v2.Artifact.Dependency.element_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cache_key', full_name='buildstream.v2.Artifact.Dependency.cache_key', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='was_workspaced', full_name='buildstream.v2.Artifact.Dependency.was_workspaced', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=583, + serialized_end=660, +) + +_ARTIFACT_LOGFILE = _descriptor.Descriptor( + name='LogFile', + full_name='buildstream.v2.Artifact.LogFile', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='buildstream.v2.Artifact.LogFile.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='digest', full_name='buildstream.v2.Artifact.LogFile.digest', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=662, + serialized_end=742, +) + +_ARTIFACT = _descriptor.Descriptor( + name='Artifact', + full_name='buildstream.v2.Artifact', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='version', full_name='buildstream.v2.Artifact.version', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_success', full_name='buildstream.v2.Artifact.build_success', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_error', full_name='buildstream.v2.Artifact.build_error', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_error_details', full_name='buildstream.v2.Artifact.build_error_details', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='strong_key', full_name='buildstream.v2.Artifact.strong_key', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='weak_key', full_name='buildstream.v2.Artifact.weak_key', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='was_workspaced', full_name='buildstream.v2.Artifact.was_workspaced', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='files', full_name='buildstream.v2.Artifact.files', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='build_deps', full_name='buildstream.v2.Artifact.build_deps', index=8, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='public_data', full_name='buildstream.v2.Artifact.public_data', index=9, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='logs', full_name='buildstream.v2.Artifact.logs', index=10, + number=11, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='buildtree', full_name='buildstream.v2.Artifact.buildtree', index=11, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_ARTIFACT_DEPENDENCY, _ARTIFACT_LOGFILE, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=136, + serialized_end=742, +) + + +_GETARTIFACTREQUEST = _descriptor.Descriptor( + name='GetArtifactRequest', + full_name='buildstream.v2.GetArtifactRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='instance_name', full_name='buildstream.v2.GetArtifactRequest.instance_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cache_key', full_name='buildstream.v2.GetArtifactRequest.cache_key', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=744, + serialized_end=806, +) + + +_UPDATEARTIFACTREQUEST = _descriptor.Descriptor( + name='UpdateArtifactRequest', + full_name='buildstream.v2.UpdateArtifactRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='instance_name', full_name='buildstream.v2.UpdateArtifactRequest.instance_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cache_key', full_name='buildstream.v2.UpdateArtifactRequest.cache_key', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='artifact', full_name='buildstream.v2.UpdateArtifactRequest.artifact', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=808, + serialized_end=917, +) + +_ARTIFACT_DEPENDENCY.containing_type = _ARTIFACT +_ARTIFACT_LOGFILE.fields_by_name['digest'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST +_ARTIFACT_LOGFILE.containing_type = _ARTIFACT +_ARTIFACT.fields_by_name['files'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST +_ARTIFACT.fields_by_name['build_deps'].message_type = _ARTIFACT_DEPENDENCY +_ARTIFACT.fields_by_name['public_data'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST +_ARTIFACT.fields_by_name['logs'].message_type = _ARTIFACT_LOGFILE +_ARTIFACT.fields_by_name['buildtree'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST +_UPDATEARTIFACTREQUEST.fields_by_name['artifact'].message_type = _ARTIFACT +DESCRIPTOR.message_types_by_name['Artifact'] = _ARTIFACT +DESCRIPTOR.message_types_by_name['GetArtifactRequest'] = _GETARTIFACTREQUEST +DESCRIPTOR.message_types_by_name['UpdateArtifactRequest'] = _UPDATEARTIFACTREQUEST +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Artifact = _reflection.GeneratedProtocolMessageType('Artifact', (_message.Message,), dict( + + Dependency = _reflection.GeneratedProtocolMessageType('Dependency', (_message.Message,), dict( + DESCRIPTOR = _ARTIFACT_DEPENDENCY, + __module__ = 'buildstream.v2.artifact_pb2' + # @@protoc_insertion_point(class_scope:buildstream.v2.Artifact.Dependency) + )) + , + + LogFile = _reflection.GeneratedProtocolMessageType('LogFile', (_message.Message,), dict( + DESCRIPTOR = _ARTIFACT_LOGFILE, + __module__ = 'buildstream.v2.artifact_pb2' + # @@protoc_insertion_point(class_scope:buildstream.v2.Artifact.LogFile) + )) + , + DESCRIPTOR = _ARTIFACT, + __module__ = 'buildstream.v2.artifact_pb2' + # @@protoc_insertion_point(class_scope:buildstream.v2.Artifact) + )) +_sym_db.RegisterMessage(Artifact) +_sym_db.RegisterMessage(Artifact.Dependency) +_sym_db.RegisterMessage(Artifact.LogFile) + +GetArtifactRequest = _reflection.GeneratedProtocolMessageType('GetArtifactRequest', (_message.Message,), dict( + DESCRIPTOR = _GETARTIFACTREQUEST, + __module__ = 'buildstream.v2.artifact_pb2' + # @@protoc_insertion_point(class_scope:buildstream.v2.GetArtifactRequest) + )) +_sym_db.RegisterMessage(GetArtifactRequest) + +UpdateArtifactRequest = _reflection.GeneratedProtocolMessageType('UpdateArtifactRequest', (_message.Message,), dict( + DESCRIPTOR = _UPDATEARTIFACTREQUEST, + __module__ = 'buildstream.v2.artifact_pb2' + # @@protoc_insertion_point(class_scope:buildstream.v2.UpdateArtifactRequest) + )) +_sym_db.RegisterMessage(UpdateArtifactRequest) + + + +_ARTIFACTSERVICE = _descriptor.ServiceDescriptor( + name='ArtifactService', + full_name='buildstream.v2.ArtifactService', + file=DESCRIPTOR, + index=0, + serialized_options=None, + serialized_start=920, + serialized_end=1101, + methods=[ + _descriptor.MethodDescriptor( + name='GetArtifact', + full_name='buildstream.v2.ArtifactService.GetArtifact', + index=0, + containing_service=None, + input_type=_GETARTIFACTREQUEST, + output_type=_ARTIFACT, + serialized_options=None, + ), + _descriptor.MethodDescriptor( + name='UpdateArtifact', + full_name='buildstream.v2.ArtifactService.UpdateArtifact', + index=1, + containing_service=None, + input_type=_UPDATEARTIFACTREQUEST, + output_type=_ARTIFACT, + serialized_options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ARTIFACTSERVICE) + +DESCRIPTOR.services_by_name['ArtifactService'] = _ARTIFACTSERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py b/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py new file mode 100644 index 000000000..d355146af --- /dev/null +++ b/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py @@ -0,0 +1,68 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +import grpc + +from buildstream._protos.buildstream.v2 import artifact_pb2 as buildstream_dot_v2_dot_artifact__pb2 + + +class ArtifactServiceStub(object): + # missing associated documentation comment in .proto file + pass + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetArtifact = channel.unary_unary( + '/buildstream.v2.ArtifactService/GetArtifact', + request_serializer=buildstream_dot_v2_dot_artifact__pb2.GetArtifactRequest.SerializeToString, + response_deserializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.FromString, + ) + self.UpdateArtifact = channel.unary_unary( + '/buildstream.v2.ArtifactService/UpdateArtifact', + request_serializer=buildstream_dot_v2_dot_artifact__pb2.UpdateArtifactRequest.SerializeToString, + response_deserializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.FromString, + ) + + +class ArtifactServiceServicer(object): + # missing associated documentation comment in .proto file + pass + + def GetArtifact(self, request, context): + """Retrieves an Artifact message + + Errors: + * `NOT_FOUND`: Artifact not found on server + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UpdateArtifact(self, request, context): + """Sets an Artifact message + + Errors: + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ArtifactServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetArtifact': grpc.unary_unary_rpc_method_handler( + servicer.GetArtifact, + request_deserializer=buildstream_dot_v2_dot_artifact__pb2.GetArtifactRequest.FromString, + response_serializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.SerializeToString, + ), + 'UpdateArtifact': grpc.unary_unary_rpc_method_handler( + servicer.UpdateArtifact, + request_deserializer=buildstream_dot_v2_dot_artifact__pb2.UpdateArtifactRequest.FromString, + response_serializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'buildstream.v2.ArtifactService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/tests/artifactcache/artifactservice.py b/tests/artifactcache/artifactservice.py new file mode 100644 index 000000000..5a7a3cdd5 --- /dev/null +++ b/tests/artifactcache/artifactservice.py @@ -0,0 +1,109 @@ +# +# Copyright (C) 2019 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk> +# +import os +import pytest +from urllib.parse import urlparse + +import grpc + +from buildstream._protos.buildstream.v2.artifact_pb2 \ + import Artifact, GetArtifactRequest, UpdateArtifactRequest +from buildstream._protos.buildstream.v2.artifact_pb2_grpc import ArtifactServiceStub +from buildstream._protos.build.bazel.remote.execution.v2 \ + import remote_execution_pb2 as re_pb2 +from buildstream._protos.build.bazel.remote.execution.v2 \ + import remote_execution_pb2_grpc as re_pb2_grpc +from buildstream import utils + +from tests.testutils.artifactshare import create_artifact_share + + +def test_artifact_get_not_found(tmpdir): + sharedir = os.path.join(str(tmpdir), "share") + with create_artifact_share(sharedir) as share: + # set up artifact service stub + url = urlparse(share.repo) + channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) + artifact_stub = ArtifactServiceStub(channel) + + # Run GetArtifact and check it throws a not found error + request = GetArtifactRequest() + request.cache_key = "@artifact/something/not_there" + try: + artifact_stub.GetArtifact(request) + except grpc.RpcError as e: + assert e.code() == grpc.StatusCode.NOT_FOUND + assert e.details() == "Artifact proto not found" + else: + assert False + +# Successfully getting the artifact +@pytest.mark.parametrize("files", ["present", "absent", "invalid"]) +def test_update_artifact(tmpdir, files): + sharedir = os.path.join(str(tmpdir), "share") + with create_artifact_share(sharedir) as share: + url = urlparse(share.repo) + channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) + artifact_stub = ArtifactServiceStub(channel) + + # initialise an artifact + artifact = Artifact() + artifact.version = 0 + artifact.build_success = True + artifact.strong_key = "abcdefghijklmnop" + artifact.files.hash = "hashymchashash" + artifact.files.size_bytes = 10 + + # put files object + if files == "present": + directory = re_pb2.Directory() + digest = share.cas.add_object(buffer=directory.SerializeToString()) + elif files == "invalid": + digest = share.cas.add_object(buffer="abcdefghijklmnop".encode("utf-8")) + elif files == "absent": + digest = utils._message_digest("abcdefghijklmnop".encode("utf-8")) + + artifact.files.CopyFrom(digest) + + # Put it in the artifact share with an UpdateArtifactRequest + request = UpdateArtifactRequest() + request.artifact.CopyFrom(artifact) + request.cache_key = "a-cache-key" + + # should return the same artifact back + if files == "present": + response = artifact_stub.UpdateArtifact(request) + assert response == artifact + else: + try: + artifact_stub.UpdateArtifact(request) + except grpc.RpcError as e: + assert e.code() == grpc.StatusCode.FAILED_PRECONDITION + if files == "absent": + assert e.details() == "Artifact files specified but no files found" + elif files == "invalid": + assert e.details() == "Artifact files specified but directory not found" + return + + # If we uploaded the artifact check GetArtifact + request = GetArtifactRequest() + request.cache_key = "a-cache-key" + + response = artifact_stub.GetArtifact(request) + assert response == artifact |