Merge branch 'raoul/965-AaaP-service' into 'master'

Artifact as a Proto: protos and service

Closes #965

See merge request BuildStream/buildstream!1259

5 files changed, 734 insertions, 1 deletions

diff --git a/buildstream/_cas/casserver.py b/buildstream/_cas/casserver.py
index 8f8ef4efe..f88db717a 100644
--- a/buildstream/_cas/casserver.py
+++ b/buildstream/_cas/casserver.py
@@ -28,12 +28,14 @@ import errno
 import threading
 
 import grpc
+from google.protobuf.message import DecodeError
 import click
 
 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
 from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
-from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
 from .._protos.google.rpc import code_pb2
+from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
+    artifact_pb2, artifact_pb2_grpc
 
 from .._exceptions import CASError
 
@@ -62,6 +64,7 @@ def create_server(repo, *, enable_push,
                   max_head_size=int(10e9),
                   min_head_size=int(2e9)):
     cas = CASCache(os.path.abspath(repo))
+    artifactdir = os.path.join(os.path.abspath(repo), 'artifacts', 'refs')
 
     # Use max_workers default from Python 3.5+
     max_workers = (os.cpu_count() or 1) * 5
@@ -81,6 +84,9 @@ def create_server(repo, *, enable_push,
     buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
         _ReferenceStorageServicer(cas, enable_push=enable_push), server)
 
+    artifact_pb2_grpc.add_ArtifactServiceServicer_to_server(
+        _ArtifactServicer(cas, artifactdir), server)
+
     return server
 
 
@@ -405,6 +411,81 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
         return response
 
 
+class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
+
+    def __init__(self, cas, artifactdir):
+        super().__init__()
+        self.cas = cas
+        self.artifactdir = artifactdir
+        os.makedirs(artifactdir, exist_ok=True)
+
+    def GetArtifact(self, request, context):
+        artifact_path = os.path.join(self.artifactdir, request.cache_key)
+        if not os.path.exists(artifact_path):
+            context.abort(grpc.StatusCode.NOT_FOUND, "Artifact proto not found")
+
+        artifact = artifact_pb2.Artifact()
+        with open(artifact_path, 'rb') as f:
+            artifact.ParseFromString(f.read())
+
+        files_digest = artifact.files
+
+        # Now update mtimes of files present.
+        try:
+            self.cas.update_tree_mtime(files_digest)
+        except FileNotFoundError:
+            os.unlink(artifact_path)
+            context.abort(grpc.StatusCode.NOT_FOUND,
+                          "Artifact files incomplete")
+        except DecodeError:
+            context.abort(grpc.StatusCode.NOT_FOUND,
+                          "Artifact files not valid")
+
+        return artifact
+
+    def UpdateArtifact(self, request, context):
+        artifact = request.artifact
+
+        # Check that the files specified are in the CAS
+        self._check_directory("files", artifact.files, context)
+
+        # Unset protocol buffers don't evaluated to False but do return empty
+        # strings, hence str()
+        if str(artifact.buildtree):
+            self._check_directory("buildtree", artifact.buildtree, context)
+
+        if str(artifact.public_data):
+            self._check_file("public data", artifact.public_data, context)
+
+        for log_file in artifact.logs:
+            self._check_file("log digest", log_file.digest, context)
+
+        # Add the artifact proto to the cas
+        artifact_path = os.path.join(self.artifactdir, request.cache_key)
+        os.makedirs(os.path.dirname(artifact_path), exist_ok=True)
+        with open(artifact_path, 'wb') as f:
+            f.write(artifact.SerializeToString())
+
+        return artifact
+
+    def _check_directory(self, name, digest, context):
+        try:
+            directory = remote_execution_pb2.Directory()
+            with open(self.cas.objpath(digest), 'rb') as f:
+                directory.ParseFromString(f.read())
+        except FileNotFoundError:
+            context.abort(grpc.StatusCode.FAILED_PRECONDITION,
+                          "Artifact {} specified but no files found".format(name))
+        except DecodeError:
+            context.abort(grpc.StatusCode.FAILED_PRECONDITION,
+                          "Artifact {} specified but directory not found".format(name))
+
+    def _check_file(self, name, digest, context):
+        if not os.path.exists(self.cas.objpath(digest)):
+            context.abort(grpc.StatusCode.FAILED_PRECONDITION,
+                          "Artifact {} specified but not found".format(name))
+
+
 def _digest_from_download_resource_name(resource_name):
     parts = resource_name.split('/')
 
diff --git a/buildstream/_protos/buildstream/v2/artifact.proto b/buildstream/_protos/buildstream/v2/artifact.proto
new file mode 100644
index 000000000..56ddbca6b
--- /dev/null
+++ b/buildstream/_protos/buildstream/v2/artifact.proto
@@ -0,0 +1,88 @@
+// Copyright 2019 Bloomberg Finance LP
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Authors
+//      Raoul Hidalgo Charman <raoul.hidalgo.charman@gmail.com>
+
+syntax = "proto3";
+
+package buildstream.v2;
+
+import "build/bazel/remote/execution/v2/remote_execution.proto";
+import "google/api/annotations.proto";
+
+service ArtifactService {
+  // Retrieves an Artifact message
+  //
+  // Errors:
+  // * `NOT_FOUND`: Artifact not found on server
+  rpc GetArtifact(GetArtifactRequest) returns (Artifact) {}
+
+  // Sets an Artifact message
+  //
+  // Errors:
+  // * `FAILED_PRECONDITION`: Files specified in upload aren't present in CAS
+  rpc UpdateArtifact(UpdateArtifactRequest) returns (Artifact) {}
+}
+
+message Artifact {
+  // This version number must always be present and can be used to
+  // further indicate presence or absence of parts of the proto at a
+  // later date.  It only needs incrementing if a change to what is
+  // *mandatory* changes.
+  int32 version = 1;
+  // Core metadata
+  bool build_success = 2;
+  string build_error = 3;  // optional
+  string build_error_details = 4;
+  string strong_key = 5;
+  string weak_key = 6;
+  bool was_workspaced = 7;
+  // digest of a Directory
+  build.bazel.remote.execution.v2.Digest files = 8;
+
+  // Information about the build dependencies
+  message Dependency {
+    string element_name = 1;
+    string cache_key = 2;
+    bool was_workspaced = 3;
+  };
+  repeated Dependency build_deps = 9;
+
+  // The public data is a yaml file which is stored into the CAS
+  // Digest is of a directory
+  build.bazel.remote.execution.v2.Digest public_data = 10;
+
+  // The logs are stored in the CAS
+  message LogFile {
+    string name = 1;
+    // digest of a file
+    build.bazel.remote.execution.v2.Digest digest = 2;
+  };
+  repeated LogFile logs = 11;  // Zero or more log files here
+
+  // digest of a directory
+  build.bazel.remote.execution.v2.Digest buildtree = 12;  // optional
+}
+
+message GetArtifactRequest {
+  string instance_name = 1;
+  string cache_key = 2;
+}
+
+message UpdateArtifactRequest {
+  string instance_name = 1;
+  string cache_key = 2;
+  Artifact artifact = 3;
+}
diff --git a/buildstream/_protos/buildstream/v2/artifact_pb2.py b/buildstream/_protos/buildstream/v2/artifact_pb2.py
new file mode 100644
index 000000000..c56d1ae8a
--- /dev/null
+++ b/buildstream/_protos/buildstream/v2/artifact_pb2.py
@@ -0,0 +1,387 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: buildstream/v2/artifact.proto
+
+import sys
+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2
+from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='buildstream/v2/artifact.proto',
+  package='buildstream.v2',
+  syntax='proto3',
+  serialized_options=None,
+  serialized_pb=_b('\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\xde\x04\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x18\x06 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x07 \x01(\x08\x12\x36\n\x05\x66iles\x18\x08 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x37\n\nbuild_deps\x18\t \x03(\x0b\x32#.buildstream.v2.Artifact.Dependency\x12<\n\x0bpublic_data\x18\n \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12.\n\x04logs\x18\x0b \x03(\x0b\x32 .buildstream.v2.Artifact.LogFile\x12:\n\tbuildtree\x18\x0c \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x1aM\n\nDependency\x12\x14\n\x0c\x65lement_name\x18\x01 \x01(\t\x12\x11\n\tcache_key\x18\x02 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x03 \x01(\x08\x1aP\n\x07LogFile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\">\n\x12GetArtifactRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x11\n\tcache_key\x18\x02 \x01(\t\"m\n\x15UpdateArtifactRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x11\n\tcache_key\x18\x02 \x01(\t\x12*\n\x08\x61rtifact\x18\x03 \x01(\x0b\x32\x18.buildstream.v2.Artifact2\xb5\x01\n\x0f\x41rtifactService\x12M\n\x0bGetArtifact\x12\".buildstream.v2.GetArtifactRequest\x1a\x18.buildstream.v2.Artifact\"\x00\x12S\n\x0eUpdateArtifact\x12%.buildstream.v2.UpdateArtifactRequest\x1a\x18.buildstream.v2.Artifact\"\x00\x62\x06proto3')
+  ,
+  dependencies=[build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.DESCRIPTOR,google_dot_api_dot_annotations__pb2.DESCRIPTOR,])
+
+
+
+
+_ARTIFACT_DEPENDENCY = _descriptor.Descriptor(
+  name='Dependency',
+  full_name='buildstream.v2.Artifact.Dependency',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='element_name', full_name='buildstream.v2.Artifact.Dependency.element_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='cache_key', full_name='buildstream.v2.Artifact.Dependency.cache_key', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='was_workspaced', full_name='buildstream.v2.Artifact.Dependency.was_workspaced', index=2,
+      number=3, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=583,
+  serialized_end=660,
+)
+
+_ARTIFACT_LOGFILE = _descriptor.Descriptor(
+  name='LogFile',
+  full_name='buildstream.v2.Artifact.LogFile',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='name', full_name='buildstream.v2.Artifact.LogFile.name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='digest', full_name='buildstream.v2.Artifact.LogFile.digest', index=1,
+      number=2, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=662,
+  serialized_end=742,
+)
+
+_ARTIFACT = _descriptor.Descriptor(
+  name='Artifact',
+  full_name='buildstream.v2.Artifact',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='version', full_name='buildstream.v2.Artifact.version', index=0,
+      number=1, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='build_success', full_name='buildstream.v2.Artifact.build_success', index=1,
+      number=2, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='build_error', full_name='buildstream.v2.Artifact.build_error', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='build_error_details', full_name='buildstream.v2.Artifact.build_error_details', index=3,
+      number=4, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='strong_key', full_name='buildstream.v2.Artifact.strong_key', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='weak_key', full_name='buildstream.v2.Artifact.weak_key', index=5,
+      number=6, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='was_workspaced', full_name='buildstream.v2.Artifact.was_workspaced', index=6,
+      number=7, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='files', full_name='buildstream.v2.Artifact.files', index=7,
+      number=8, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='build_deps', full_name='buildstream.v2.Artifact.build_deps', index=8,
+      number=9, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='public_data', full_name='buildstream.v2.Artifact.public_data', index=9,
+      number=10, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='logs', full_name='buildstream.v2.Artifact.logs', index=10,
+      number=11, type=11, cpp_type=10, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='buildtree', full_name='buildstream.v2.Artifact.buildtree', index=11,
+      number=12, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[_ARTIFACT_DEPENDENCY, _ARTIFACT_LOGFILE, ],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=136,
+  serialized_end=742,
+)
+
+
+_GETARTIFACTREQUEST = _descriptor.Descriptor(
+  name='GetArtifactRequest',
+  full_name='buildstream.v2.GetArtifactRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='instance_name', full_name='buildstream.v2.GetArtifactRequest.instance_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='cache_key', full_name='buildstream.v2.GetArtifactRequest.cache_key', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=744,
+  serialized_end=806,
+)
+
+
+_UPDATEARTIFACTREQUEST = _descriptor.Descriptor(
+  name='UpdateArtifactRequest',
+  full_name='buildstream.v2.UpdateArtifactRequest',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='instance_name', full_name='buildstream.v2.UpdateArtifactRequest.instance_name', index=0,
+      number=1, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='cache_key', full_name='buildstream.v2.UpdateArtifactRequest.cache_key', index=1,
+      number=2, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='artifact', full_name='buildstream.v2.UpdateArtifactRequest.artifact', index=2,
+      number=3, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=808,
+  serialized_end=917,
+)
+
+_ARTIFACT_DEPENDENCY.containing_type = _ARTIFACT
+_ARTIFACT_LOGFILE.fields_by_name['digest'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST
+_ARTIFACT_LOGFILE.containing_type = _ARTIFACT
+_ARTIFACT.fields_by_name['files'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST
+_ARTIFACT.fields_by_name['build_deps'].message_type = _ARTIFACT_DEPENDENCY
+_ARTIFACT.fields_by_name['public_data'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST
+_ARTIFACT.fields_by_name['logs'].message_type = _ARTIFACT_LOGFILE
+_ARTIFACT.fields_by_name['buildtree'].message_type = build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2._DIGEST
+_UPDATEARTIFACTREQUEST.fields_by_name['artifact'].message_type = _ARTIFACT
+DESCRIPTOR.message_types_by_name['Artifact'] = _ARTIFACT
+DESCRIPTOR.message_types_by_name['GetArtifactRequest'] = _GETARTIFACTREQUEST
+DESCRIPTOR.message_types_by_name['UpdateArtifactRequest'] = _UPDATEARTIFACTREQUEST
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Artifact = _reflection.GeneratedProtocolMessageType('Artifact', (_message.Message,), dict(
+
+  Dependency = _reflection.GeneratedProtocolMessageType('Dependency', (_message.Message,), dict(
+    DESCRIPTOR = _ARTIFACT_DEPENDENCY,
+    __module__ = 'buildstream.v2.artifact_pb2'
+    # @@protoc_insertion_point(class_scope:buildstream.v2.Artifact.Dependency)
+    ))
+  ,
+
+  LogFile = _reflection.GeneratedProtocolMessageType('LogFile', (_message.Message,), dict(
+    DESCRIPTOR = _ARTIFACT_LOGFILE,
+    __module__ = 'buildstream.v2.artifact_pb2'
+    # @@protoc_insertion_point(class_scope:buildstream.v2.Artifact.LogFile)
+    ))
+  ,
+  DESCRIPTOR = _ARTIFACT,
+  __module__ = 'buildstream.v2.artifact_pb2'
+  # @@protoc_insertion_point(class_scope:buildstream.v2.Artifact)
+  ))
+_sym_db.RegisterMessage(Artifact)
+_sym_db.RegisterMessage(Artifact.Dependency)
+_sym_db.RegisterMessage(Artifact.LogFile)
+
+GetArtifactRequest = _reflection.GeneratedProtocolMessageType('GetArtifactRequest', (_message.Message,), dict(
+  DESCRIPTOR = _GETARTIFACTREQUEST,
+  __module__ = 'buildstream.v2.artifact_pb2'
+  # @@protoc_insertion_point(class_scope:buildstream.v2.GetArtifactRequest)
+  ))
+_sym_db.RegisterMessage(GetArtifactRequest)
+
+UpdateArtifactRequest = _reflection.GeneratedProtocolMessageType('UpdateArtifactRequest', (_message.Message,), dict(
+  DESCRIPTOR = _UPDATEARTIFACTREQUEST,
+  __module__ = 'buildstream.v2.artifact_pb2'
+  # @@protoc_insertion_point(class_scope:buildstream.v2.UpdateArtifactRequest)
+  ))
+_sym_db.RegisterMessage(UpdateArtifactRequest)
+
+
+
+_ARTIFACTSERVICE = _descriptor.ServiceDescriptor(
+  name='ArtifactService',
+  full_name='buildstream.v2.ArtifactService',
+  file=DESCRIPTOR,
+  index=0,
+  serialized_options=None,
+  serialized_start=920,
+  serialized_end=1101,
+  methods=[
+  _descriptor.MethodDescriptor(
+    name='GetArtifact',
+    full_name='buildstream.v2.ArtifactService.GetArtifact',
+    index=0,
+    containing_service=None,
+    input_type=_GETARTIFACTREQUEST,
+    output_type=_ARTIFACT,
+    serialized_options=None,
+  ),
+  _descriptor.MethodDescriptor(
+    name='UpdateArtifact',
+    full_name='buildstream.v2.ArtifactService.UpdateArtifact',
+    index=1,
+    containing_service=None,
+    input_type=_UPDATEARTIFACTREQUEST,
+    output_type=_ARTIFACT,
+    serialized_options=None,
+  ),
+])
+_sym_db.RegisterServiceDescriptor(_ARTIFACTSERVICE)
+
+DESCRIPTOR.services_by_name['ArtifactService'] = _ARTIFACTSERVICE
+
+# @@protoc_insertion_point(module_scope)
diff --git a/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py b/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py
new file mode 100644
index 000000000..d355146af
--- /dev/null
+++ b/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py
@@ -0,0 +1,68 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+import grpc
+
+from buildstream._protos.buildstream.v2 import artifact_pb2 as buildstream_dot_v2_dot_artifact__pb2
+
+
+class ArtifactServiceStub(object):
+  # missing associated documentation comment in .proto file
+  pass
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.GetArtifact = channel.unary_unary(
+        '/buildstream.v2.ArtifactService/GetArtifact',
+        request_serializer=buildstream_dot_v2_dot_artifact__pb2.GetArtifactRequest.SerializeToString,
+        response_deserializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.FromString,
+        )
+    self.UpdateArtifact = channel.unary_unary(
+        '/buildstream.v2.ArtifactService/UpdateArtifact',
+        request_serializer=buildstream_dot_v2_dot_artifact__pb2.UpdateArtifactRequest.SerializeToString,
+        response_deserializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.FromString,
+        )
+
+
+class ArtifactServiceServicer(object):
+  # missing associated documentation comment in .proto file
+  pass
+
+  def GetArtifact(self, request, context):
+    """Retrieves an Artifact message
+
+    Errors:
+    * `NOT_FOUND`: Artifact not found on server
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def UpdateArtifact(self, request, context):
+    """Sets an Artifact message
+
+    Errors:
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_ArtifactServiceServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'GetArtifact': grpc.unary_unary_rpc_method_handler(
+          servicer.GetArtifact,
+          request_deserializer=buildstream_dot_v2_dot_artifact__pb2.GetArtifactRequest.FromString,
+          response_serializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.SerializeToString,
+      ),
+      'UpdateArtifact': grpc.unary_unary_rpc_method_handler(
+          servicer.UpdateArtifact,
+          request_deserializer=buildstream_dot_v2_dot_artifact__pb2.UpdateArtifactRequest.FromString,
+          response_serializer=buildstream_dot_v2_dot_artifact__pb2.Artifact.SerializeToString,
+      ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'buildstream.v2.ArtifactService', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))
diff --git a/tests/artifactcache/artifactservice.py b/tests/artifactcache/artifactservice.py
new file mode 100644
index 000000000..5a7a3cdd5
--- /dev/null
+++ b/tests/artifactcache/artifactservice.py
@@ -0,0 +1,109 @@
+#
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+import os
+import pytest
+from urllib.parse import urlparse
+
+import grpc
+
+from buildstream._protos.buildstream.v2.artifact_pb2 \
+    import Artifact, GetArtifactRequest, UpdateArtifactRequest
+from buildstream._protos.buildstream.v2.artifact_pb2_grpc import ArtifactServiceStub
+from buildstream._protos.build.bazel.remote.execution.v2 \
+    import remote_execution_pb2 as re_pb2
+from buildstream._protos.build.bazel.remote.execution.v2 \
+    import remote_execution_pb2_grpc as re_pb2_grpc
+from buildstream import utils
+
+from tests.testutils.artifactshare import create_artifact_share
+
+
+def test_artifact_get_not_found(tmpdir):
+    sharedir = os.path.join(str(tmpdir), "share")
+    with create_artifact_share(sharedir) as share:
+        # set up artifact service stub
+        url = urlparse(share.repo)
+        channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port))
+        artifact_stub = ArtifactServiceStub(channel)
+
+        # Run GetArtifact and check it throws a not found error
+        request = GetArtifactRequest()
+        request.cache_key = "@artifact/something/not_there"
+        try:
+            artifact_stub.GetArtifact(request)
+        except grpc.RpcError as e:
+            assert e.code() == grpc.StatusCode.NOT_FOUND
+            assert e.details() == "Artifact proto not found"
+        else:
+            assert False
+
+# Successfully getting the artifact
+@pytest.mark.parametrize("files", ["present", "absent", "invalid"])
+def test_update_artifact(tmpdir, files):
+    sharedir = os.path.join(str(tmpdir), "share")
+    with create_artifact_share(sharedir) as share:
+        url = urlparse(share.repo)
+        channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port))
+        artifact_stub = ArtifactServiceStub(channel)
+
+        # initialise an artifact
+        artifact = Artifact()
+        artifact.version = 0
+        artifact.build_success = True
+        artifact.strong_key = "abcdefghijklmnop"
+        artifact.files.hash = "hashymchashash"
+        artifact.files.size_bytes = 10
+
+        # put files object
+        if files == "present":
+            directory = re_pb2.Directory()
+            digest = share.cas.add_object(buffer=directory.SerializeToString())
+        elif files == "invalid":
+            digest = share.cas.add_object(buffer="abcdefghijklmnop".encode("utf-8"))
+        elif files == "absent":
+            digest = utils._message_digest("abcdefghijklmnop".encode("utf-8"))
+
+        artifact.files.CopyFrom(digest)
+
+        # Put it in the artifact share with an UpdateArtifactRequest
+        request = UpdateArtifactRequest()
+        request.artifact.CopyFrom(artifact)
+        request.cache_key = "a-cache-key"
+
+        # should return the same artifact back
+        if files == "present":
+            response = artifact_stub.UpdateArtifact(request)
+            assert response == artifact
+        else:
+            try:
+                artifact_stub.UpdateArtifact(request)
+            except grpc.RpcError as e:
+                assert e.code() == grpc.StatusCode.FAILED_PRECONDITION
+                if files == "absent":
+                    assert e.details() == "Artifact files specified but no files found"
+                elif files == "invalid":
+                    assert e.details() == "Artifact files specified but directory not found"
+                return
+
+        # If we uploaded the artifact check GetArtifact
+        request = GetArtifactRequest()
+        request.cache_key = "a-cache-key"
+
+        response = artifact_stub.GetArtifact(request)
+        assert response == artifact