Merge branch 'jennis/bst_artifact_show' into 'master'

Introduce `bst artifact show`

See merge request BuildStream/buildstream!1560

10 files changed, 261 insertions, 4 deletions

diff --git a/NEWS b/NEWS
index b47a70028..72bed3645 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,11 @@
 buildstream 1.3.1
 =================
 
+  o Added `bst artifact show` subcommand which shows the cached status
+    of an artifact. If project/user remotes are available, they are checked
+    for the target elements (and their deps, if specified). Artifacts available
+    in remotes are displayed as "available".
+
   o BREAKING CHANGE: The project name of dependencies have been included when
     calculating the cache key. This was required to keep inline with the
     artifact proto. Additionally, for strict mode, the name of dependencies have
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index f92a7c84f..3357f986a 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -383,6 +383,31 @@ class ArtifactCache(BaseCache):
 
         return remote_missing_blobs_list
 
+    # check_remotes_for_element()
+    #
+    # Check if the element is available in any of the remotes
+    #
+    # Args:
+    #    element (Element): The element to check
+    #
+    # Returns:
+    #    (bool): True if the element is available remotely
+    #
+    def check_remotes_for_element(self, element):
+        # If there are no remotes
+        if not self._remotes:
+            return False
+
+        project = element._get_project()
+        ref = element.get_artifact_name()
+        for remote in self._remotes[project]:
+            remote.init()
+
+            if self._query_remote(ref, remote):
+                return True
+
+        return False
+
     ################################################
     #             Local Private Methods            #
     ################################################
@@ -520,3 +545,25 @@ class ArtifactCache(BaseCache):
             f.write(artifact.SerializeToString())
 
         return True
+
+    # _query_remote()
+    #
+    # Args:
+    #    ref (str): The artifact ref
+    #    remote (ArtifactRemote): The remote we want to check
+    #
+    # Returns:
+    #    (bool): True if the ref exists in the remote, False otherwise.
+    #
+    def _query_remote(self, ref, remote):
+        request = artifact_pb2.GetArtifactRequest()
+        request.cache_key = ref
+        try:
+            artifact_service = artifact_pb2_grpc.ArtifactServiceStub(remote.channel)
+            artifact_service.GetArtifact(request)
+        except grpc.RpcError as e:
+            if e.code() != grpc.StatusCode.NOT_FOUND:
+                raise ArtifactError("Error when querying: {}".format(e.details()))
+            return False
+
+        return True
diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py
index ab26d32c7..183429a77 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -91,9 +91,6 @@ class CASRemote():
 
     def init(self):
         if not self._initialized:
-            # gRPC doesn't support fork without exec, which is used in the main process.
-            assert not utils._is_main_process()
-
             server_cert_bytes = None
             client_key_bytes = None
             client_cert_bytes = None
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 5b2e60d49..d02dd4258 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -1004,6 +1004,23 @@ def artifact():
     # they are not somehow escaped.
 
 
+#############################################################
+#                    Artifact show Command                  #
+#############################################################
+@artifact.command(name='show', short_help="Show the cached state of artifacts")
+@click.option('--deps', '-d', default='none',
+              type=click.Choice(['build', 'run', 'all', 'none']),
+              help='The dependencies we also want to show (default: none)')
+@click.argument('artifacts', type=click.Path(), nargs=-1)
+@click.pass_obj
+def artifact_show(app, deps, artifacts):
+    """show the cached state of artifacts"""
+    with app.initialized():
+        targets = app.stream.artifact_show(artifacts, selection=deps)
+        click.echo(app.logger.show_state_of_artifacts(targets))
+        sys.exit(0)
+
+
 #####################################################################
 #                     Artifact Checkout Command                     #
 #####################################################################
diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py
index 955680f9b..e8299868c 100644
--- a/src/buildstream/_frontend/widget.py
+++ b/src/buildstream/_frontend/widget.py
@@ -827,3 +827,40 @@ class LogLine(Widget):
             text += '\n'
 
         return text
+
+    # show_state_of_artifacts()
+    #
+    # Show the cached status of artifacts
+    #
+    # Example output:
+    #
+    #    "cached      foo.bst"      <- If cached locally
+    #    "failed      foo.bst"      <- If cached locally as a failure
+    #    "available   foo.bst"      <- If available to download from a remote
+    #    "not cached  foo.bst"      <- If not cached/available remotely.
+    #
+    # Note that artifact names may also be displayed here.
+    #
+    # Args:
+    #    targets (list [Element]): Elements (or ArtifactElements) we wish to show the
+    #                              cached status of
+    #
+    def show_state_of_artifacts(self, targets):
+        report = ''
+        p = Profile()
+        for element in targets:
+            line = '%{state: >12} %{name}'
+            line = p.fmt_subst(line, 'name', element.name, fg='yellow')
+
+            if element._cached_success():
+                line = p.fmt_subst(line, 'state', "cached", fg='magenta')
+            elif element._cached():
+                line = p.fmt_subst(line, 'state', "failed", fg='red')
+            elif element._cached_remotely():
+                line = p.fmt_subst(line, 'state', "available", fg='green')
+            else:
+                line = p.fmt_subst(line, 'state', "not cached", fg='bright_red')
+
+            report += line + '\n'
+
+        return report
diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py
index 7cf4abbe3..4b0c6ad94 100644
--- a/src/buildstream/_pipeline.py
+++ b/src/buildstream/_pipeline.py
@@ -154,6 +154,18 @@ class Pipeline():
                 # dependencies.
                 element._update_ready_for_runtime_and_cached()
 
+    # check_remotes()
+    #
+    # Check if the target artifact is cached in any of the available remotes
+    #
+    # Args:
+    #    targets (list [Element]): The list of element targets
+    #
+    def check_remotes(self, targets):
+        with self._context.messenger.timed_activity("Querying remotes for cached status", silent_nested=True):
+            for element in targets:
+                element._cached_remotely()
+
     # dependencies()
     #
     # Generator function to iterate over elements and optionally
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 58d4a0ebd..11f428aaf 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -613,6 +613,33 @@ class Stream():
                     with tarfile.open(location, mode=mode) as tf:
                         virdir.export_to_tar(tf, '.')
 
+    # artifact_show()
+    #
+    # Show cached artifacts
+    #
+    # Args:
+    #    targets (str): Targets to show the cached state of
+    #
+    def artifact_show(self, targets, *,
+                      selection=PipelineSelection.NONE):
+        # Obtain list of Element and/or ArtifactElement objects
+        target_objects = self.load_selection(targets,
+                                             selection=selection,
+                                             use_artifact_config=True,
+                                             load_refs=True)
+
+        if self._artifacts.has_fetch_remotes():
+            self._context.disable_fork()
+            self._pipeline.check_remotes(target_objects)
+
+        # XXX: We need to set the name of an ArtifactElement to its ref in order
+        #      to display the expected result in the frontend
+        for obj in target_objects:
+            if isinstance(obj, ArtifactElement):
+                obj.name = obj.get_artifact_name()
+
+        return target_objects
+
     # artifact_log()
     #
     # Show the full log of an artifact
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index c44af942b..10c8320fa 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -219,6 +219,7 @@ class Element(Plugin):
         self.__updated_strict_cache_keys_of_rdeps = False  # Whether we've updated strict cache keys of rdeps
         self.__ready_for_runtime = False        # Whether the element and its runtime dependencies have cache keys
         self.__ready_for_runtime_and_cached = False  # Whether all runtime deps are cached, as well as the element
+        self.__cached_remotely = None           # Whether the element is cached remotely
         self.__sources = []                     # List of Sources
         self.__weak_cache_key = None            # Our cached weak cache key
         self.__strict_cache_key = None          # Our cached cache key for strict builds
@@ -1055,6 +1056,16 @@ class Element(Plugin):
 
         return self.__artifact.cached()
 
+    # _cached_remotely():
+    #
+    # Returns:
+    #    (bool): Whether this element is present in a remote cache
+    #
+    def _cached_remotely(self):
+        if self.__cached_remotely is None:
+            self.__cached_remotely = self.__artifacts.check_remotes_for_element(self)
+        return self.__cached_remotely
+
     # _get_build_result():
     #
     # Returns:
diff --git a/tests/frontend/artifact.py b/tests/frontend/artifact.py
index f48807ef6..cbc9ab022 100644
--- a/tests/frontend/artifact.py
+++ b/tests/frontend/artifact.py
@@ -25,8 +25,8 @@ import os
 import pytest
 
 from buildstream.element import _get_normal_name
-from buildstream.testing import cli  # pylint: disable=unused-import
 from buildstream._exceptions import ErrorDomain
+from buildstream.testing import cli  # pylint: disable=unused-import
 from tests.testutils import create_artifact_share
 
 
@@ -369,3 +369,106 @@ def test_artifact_delete_artifact_with_deps_all_fails(cli, tmpdir, datafiles):
     result.assert_main_error(ErrorDomain.STREAM, None)
 
     assert "Error: '--deps all' is not supported for artifact refs" in result.stderr
+
+
+# Test artifact show
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_show_element_name(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    element = 'target.bst'
+
+    result = cli.run(project=project, args=['artifact', 'show', element])
+    result.assert_success()
+    assert 'not cached {}'.format(element) in result.output
+
+    result = cli.run(project=project, args=['build', element])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['artifact', 'show', element])
+    result.assert_success()
+    assert 'cached {}'.format(element) in result.output
+
+
+# Test artifact show on a failed element
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_show_failed_element(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    element = 'manual.bst'
+
+    result = cli.run(project=project, args=['artifact', 'show', element])
+    result.assert_success()
+    assert 'not cached {}'.format(element) in result.output
+
+    result = cli.run(project=project, args=['build', element])
+    result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command')
+
+    result = cli.run(project=project, args=['artifact', 'show', element])
+    result.assert_success()
+    assert 'failed {}'.format(element) in result.output
+
+
+# Test artifact show with a deleted dependency
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_show_element_missing_deps(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    element = 'target.bst'
+    dependency = 'import-bin.bst'
+
+    result = cli.run(project=project, args=['build', element])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['artifact', 'delete', dependency])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['artifact', 'show', '--deps', 'all', element])
+    result.assert_success()
+    assert 'not cached {}'.format(dependency) in result.output
+    assert 'cached {}'.format(element) in result.output
+
+
+# Test artifact show with artifact ref
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_show_artifact_ref(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    element = 'target.bst'
+
+    result = cli.run(project=project, args=['build', element])
+    result.assert_success()
+
+    cache_key = cli.get_element_key(project, element)
+    artifact_ref = 'test/target/' + cache_key
+
+    result = cli.run(project=project, args=['artifact', 'show', artifact_ref])
+    result.assert_success()
+    assert 'cached {}'.format(artifact_ref) in result.output
+
+
+# Test artifact show artifact in remote
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    element = 'target.bst'
+
+    # Set up remote and local shares
+    local_cache = os.path.join(str(tmpdir), 'artifacts')
+    with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote:
+        cli.configure({
+            'artifacts': {'url': remote.repo, 'push': True},
+            'cachedir': local_cache,
+        })
+
+        # Build the element
+        result = cli.run(project=project, args=['build', element])
+        result.assert_success()
+
+        # Make sure it's in the share
+        assert remote.has_artifact(cli.get_artifact_name(project, 'test', element))
+
+        # Delete the artifact from the local cache
+        result = cli.run(project=project, args=['artifact', 'delete', element])
+        result.assert_success()
+        assert cli.get_element_state(project, element) != 'cached'
+
+        result = cli.run(project=project, args=['artifact', 'show', element])
+        result.assert_success()
+        assert 'available {}'.format(element) in result.output
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index e9fa25b73..a254d9082 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -66,6 +66,7 @@ ARTIFACT_COMMANDS = [
     'pull ',
     'log ',
     'list-contents ',
+    'show ',
 ]
 
 WORKSPACE_COMMANDS = [