Merge branch 'juerg/shell' into 'master'

Simplify `bst shell` implementation

See merge request BuildStream/buildstream!2084

34 files changed, 129 insertions, 277 deletions

diff --git a/NEWS b/NEWS
index b4d2648b1..1e8314040 100644
--- a/NEWS
+++ b/NEWS
@@ -23,6 +23,13 @@ Core
   * BREAKING CHANGE: Changed ScriptElement.layout_add() API to take Element instances
                      in place of Element names
 
+CLI
+---
+
+  o BREAKING CHANGE: `bst shell --use-buildtree` is now a boolean flag.
+    `--use-buildtree=ask` and `--use-buildtree=try` are no longer supported.
+
+
 ==================
 buildstream 1.93.5
 ==================
diff --git a/man/bst-artifact-checkout.1 b/man/bst-artifact-checkout.1
index d9d62d875..d17bba65f 100644
--- a/man/bst-artifact-checkout.1
+++ b/man/bst-artifact-checkout.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT CHECKOUT" "1" "2020-08-13" "" "bst artifact checkout Manual"
+.TH "BST ARTIFACT CHECKOUT" "1" "2020-10-14" "" "bst artifact checkout Manual"
 .SH NAME
 bst\-artifact\-checkout \- Checkout contents of an artifact
 .SH SYNOPSIS
diff --git a/man/bst-artifact-delete.1 b/man/bst-artifact-delete.1
index 76c383d6b..c4df14ba2 100644
--- a/man/bst-artifact-delete.1
+++ b/man/bst-artifact-delete.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT DELETE" "1" "2020-08-13" "" "bst artifact delete Manual"
+.TH "BST ARTIFACT DELETE" "1" "2020-10-14" "" "bst artifact delete Manual"
 .SH NAME
 bst\-artifact\-delete \- Remove artifacts from the local cache
 .SH SYNOPSIS
diff --git a/man/bst-artifact-list-contents.1 b/man/bst-artifact-list-contents.1
index e3d5f1487..f42c049cf 100644
--- a/man/bst-artifact-list-contents.1
+++ b/man/bst-artifact-list-contents.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT LIST-CONTENTS" "1" "2020-08-13" "" "bst artifact list-contents Manual"
+.TH "BST ARTIFACT LIST-CONTENTS" "1" "2020-10-14" "" "bst artifact list-contents Manual"
 .SH NAME
 bst\-artifact\-list-contents \- List the contents of an artifact
 .SH SYNOPSIS
diff --git a/man/bst-artifact-log.1 b/man/bst-artifact-log.1
index 769907e21..093f26771 100644
--- a/man/bst-artifact-log.1
+++ b/man/bst-artifact-log.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT LOG" "1" "2020-08-13" "" "bst artifact log Manual"
+.TH "BST ARTIFACT LOG" "1" "2020-10-14" "" "bst artifact log Manual"
 .SH NAME
 bst\-artifact\-log \- Show logs of artifacts
 .SH SYNOPSIS
diff --git a/man/bst-artifact-pull.1 b/man/bst-artifact-pull.1
index c31b1b4c9..14bba67f1 100644
--- a/man/bst-artifact-pull.1
+++ b/man/bst-artifact-pull.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT PULL" "1" "2020-08-13" "" "bst artifact pull Manual"
+.TH "BST ARTIFACT PULL" "1" "2020-10-14" "" "bst artifact pull Manual"
 .SH NAME
 bst\-artifact\-pull \- Pull a built artifact
 .SH SYNOPSIS
diff --git a/man/bst-artifact-push.1 b/man/bst-artifact-push.1
index cb312c9e5..551ca3c22 100644
--- a/man/bst-artifact-push.1
+++ b/man/bst-artifact-push.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT PUSH" "1" "2020-08-13" "" "bst artifact push Manual"
+.TH "BST ARTIFACT PUSH" "1" "2020-10-14" "" "bst artifact push Manual"
 .SH NAME
 bst\-artifact\-push \- Push a built artifact
 .SH SYNOPSIS
diff --git a/man/bst-artifact-server.1 b/man/bst-artifact-server.1
index d7120a1a1..757f6b9e1 100644
--- a/man/bst-artifact-server.1
+++ b/man/bst-artifact-server.1
@@ -1,4 +1,4 @@
-.TH "BST-ARTIFACT-SERVER" "1" "2020-08-13" "" "bst-artifact-server Manual"
+.TH "BST-ARTIFACT-SERVER" "1" "2020-10-14" "" "bst-artifact-server Manual"
 .SH NAME
 bst-artifact-server \- CAS Artifact Server
 .SH SYNOPSIS
diff --git a/man/bst-artifact-show.1 b/man/bst-artifact-show.1
index fc59ba320..8899dc3ad 100644
--- a/man/bst-artifact-show.1
+++ b/man/bst-artifact-show.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT SHOW" "1" "2020-08-13" "" "bst artifact show Manual"
+.TH "BST ARTIFACT SHOW" "1" "2020-10-14" "" "bst artifact show Manual"
 .SH NAME
 bst\-artifact\-show \- Show the cached state of artifacts
 .SH SYNOPSIS
diff --git a/man/bst-artifact.1 b/man/bst-artifact.1
index 3cb170557..9506ff7f9 100644
--- a/man/bst-artifact.1
+++ b/man/bst-artifact.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT" "1" "2020-08-13" "" "bst artifact Manual"
+.TH "BST ARTIFACT" "1" "2020-10-14" "" "bst artifact Manual"
 .SH NAME
 bst\-artifact \- Manipulate cached artifacts.
 .SH SYNOPSIS
diff --git a/man/bst-build.1 b/man/bst-build.1
index a5a25536b..d0c271f05 100644
--- a/man/bst-build.1
+++ b/man/bst-build.1
@@ -1,4 +1,4 @@
-.TH "BST BUILD" "1" "2020-08-13" "" "bst build Manual"
+.TH "BST BUILD" "1" "2020-10-14" "" "bst build Manual"
 .SH NAME
 bst\-build \- Build elements in a pipeline
 .SH SYNOPSIS
diff --git a/man/bst-help.1 b/man/bst-help.1
index efd495ba1..58837e34b 100644
--- a/man/bst-help.1
+++ b/man/bst-help.1
@@ -1,4 +1,4 @@
-.TH "BST HELP" "1" "2020-08-13" "" "bst help Manual"
+.TH "BST HELP" "1" "2020-10-14" "" "bst help Manual"
 .SH NAME
 bst\-help \- Print usage information
 .SH SYNOPSIS
diff --git a/man/bst-init.1 b/man/bst-init.1
index 433b7de67..2bf2e836d 100644
--- a/man/bst-init.1
+++ b/man/bst-init.1
@@ -1,4 +1,4 @@
-.TH "BST INIT" "1" "2020-08-13" "" "bst init Manual"
+.TH "BST INIT" "1" "2020-10-14" "" "bst init Manual"
 .SH NAME
 bst\-init \- Initialize a new BuildStream project
 .SH SYNOPSIS
diff --git a/man/bst-shell.1 b/man/bst-shell.1
index 925ae1f56..40636e643 100644
--- a/man/bst-shell.1
+++ b/man/bst-shell.1
@@ -1,4 +1,4 @@
-.TH "BST SHELL" "1" "2020-08-13" "" "bst shell Manual"
+.TH "BST SHELL" "1" "2020-10-14" "" "bst shell Manual"
 .SH NAME
 bst\-shell \- Shell into an element's sandbox environment
 .SH SYNOPSIS
@@ -36,8 +36,8 @@ Mount a file or directory into the sandbox
 \fB\-\-isolate\fP
 Create an isolated build sandbox
 .TP
-\fB\-t,\fP \-\-use\-buildtree [ask|try|always|never]
-Stage a buildtree. If `always` is set, will always fail to build if a buildtree is not available. --pull and pull-buildtrees configuration is needed if trying to query for remotely cached buildtrees.  [default: ask]
+\fB\-t,\fP \-\-use\-buildtree
+Stage a buildtree. Will fail if a buildtree is not available. --pull and pull-buildtrees configuration is needed if trying to query for remotely cached buildtrees.
 .TP
 \fB\-\-pull\fP
 Attempt to pull missing or incomplete artifacts
diff --git a/man/bst-show.1 b/man/bst-show.1
index eaee58be6..1ce152f94 100644
--- a/man/bst-show.1
+++ b/man/bst-show.1
@@ -1,4 +1,4 @@
-.TH "BST SHOW" "1" "2020-08-13" "" "bst show Manual"
+.TH "BST SHOW" "1" "2020-10-14" "" "bst show Manual"
 .SH NAME
 bst\-show \- Show elements in the pipeline
 .SH SYNOPSIS
diff --git a/man/bst-source-checkout.1 b/man/bst-source-checkout.1
index f450d97f3..134593114 100644
--- a/man/bst-source-checkout.1
+++ b/man/bst-source-checkout.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE CHECKOUT" "1" "2020-08-13" "" "bst source checkout Manual"
+.TH "BST SOURCE CHECKOUT" "1" "2020-10-14" "" "bst source checkout Manual"
 .SH NAME
 bst\-source\-checkout \- Checkout sources of an element
 .SH SYNOPSIS
diff --git a/man/bst-source-fetch.1 b/man/bst-source-fetch.1
index 4fc2633a2..35a94c6e5 100644
--- a/man/bst-source-fetch.1
+++ b/man/bst-source-fetch.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE FETCH" "1" "2020-08-13" "" "bst source fetch Manual"
+.TH "BST SOURCE FETCH" "1" "2020-10-14" "" "bst source fetch Manual"
 .SH NAME
 bst\-source\-fetch \- Fetch sources in a pipeline
 .SH SYNOPSIS
diff --git a/man/bst-source-push.1 b/man/bst-source-push.1
index d517e74b5..531a1e461 100644
--- a/man/bst-source-push.1
+++ b/man/bst-source-push.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE PUSH" "1" "2020-08-13" "" "bst source push Manual"
+.TH "BST SOURCE PUSH" "1" "2020-10-14" "" "bst source push Manual"
 .SH NAME
 bst\-source\-push \- Push sources in a pipeline
 .SH SYNOPSIS
diff --git a/man/bst-source-track.1 b/man/bst-source-track.1
index 3cd3b17b5..db8631e79 100644
--- a/man/bst-source-track.1
+++ b/man/bst-source-track.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE TRACK" "1" "2020-08-13" "" "bst source track Manual"
+.TH "BST SOURCE TRACK" "1" "2020-10-14" "" "bst source track Manual"
 .SH NAME
 bst\-source\-track \- Track new source references
 .SH SYNOPSIS
diff --git a/man/bst-source.1 b/man/bst-source.1
index 94a268a57..0f308aa66 100644
--- a/man/bst-source.1
+++ b/man/bst-source.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE" "1" "2020-08-13" "" "bst source Manual"
+.TH "BST SOURCE" "1" "2020-10-14" "" "bst source Manual"
 .SH NAME
 bst\-source \- Manipulate sources for an element
 .SH SYNOPSIS
diff --git a/man/bst-workspace-close.1 b/man/bst-workspace-close.1
index 6abce3485..24fc7a4d9 100644
--- a/man/bst-workspace-close.1
+++ b/man/bst-workspace-close.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE CLOSE" "1" "2020-08-13" "" "bst workspace close Manual"
+.TH "BST WORKSPACE CLOSE" "1" "2020-10-14" "" "bst workspace close Manual"
 .SH NAME
 bst\-workspace\-close \- Close workspaces
 .SH SYNOPSIS
diff --git a/man/bst-workspace-list.1 b/man/bst-workspace-list.1
index 4e7aacd52..04ff5f323 100644
--- a/man/bst-workspace-list.1
+++ b/man/bst-workspace-list.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE LIST" "1" "2020-08-13" "" "bst workspace list Manual"
+.TH "BST WORKSPACE LIST" "1" "2020-10-14" "" "bst workspace list Manual"
 .SH NAME
 bst\-workspace\-list \- List open workspaces
 .SH SYNOPSIS
diff --git a/man/bst-workspace-open.1 b/man/bst-workspace-open.1
index 900205a7a..ab3d7f68e 100644
--- a/man/bst-workspace-open.1
+++ b/man/bst-workspace-open.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE OPEN" "1" "2020-08-13" "" "bst workspace open Manual"
+.TH "BST WORKSPACE OPEN" "1" "2020-10-14" "" "bst workspace open Manual"
 .SH NAME
 bst\-workspace\-open \- Open a new workspace
 .SH SYNOPSIS
diff --git a/man/bst-workspace-reset.1 b/man/bst-workspace-reset.1
index 3a73b473b..63e31cef1 100644
--- a/man/bst-workspace-reset.1
+++ b/man/bst-workspace-reset.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE RESET" "1" "2020-08-13" "" "bst workspace reset Manual"
+.TH "BST WORKSPACE RESET" "1" "2020-10-14" "" "bst workspace reset Manual"
 .SH NAME
 bst\-workspace\-reset \- Reset a workspace to its original state
 .SH SYNOPSIS
diff --git a/man/bst-workspace.1 b/man/bst-workspace.1
index 89af0b819..a4c931d5c 100644
--- a/man/bst-workspace.1
+++ b/man/bst-workspace.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE" "1" "2020-08-13" "" "bst workspace Manual"
+.TH "BST WORKSPACE" "1" "2020-10-14" "" "bst workspace Manual"
 .SH NAME
 bst\-workspace \- Manipulate developer workspaces
 .SH SYNOPSIS
diff --git a/man/bst.1 b/man/bst.1
index d6d7a422d..4c202ffb8 100644
--- a/man/bst.1
+++ b/man/bst.1
@@ -1,4 +1,4 @@
-.TH "BST" "1" "2020-08-13" "" "bst Manual"
+.TH "BST" "1" "2020-10-14" "" "bst Manual"
 .SH NAME
 bst \- Build and manipulate BuildStream projects...
 .SH SYNOPSIS
diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py
index b25a421c3..59bc3513b 100644
--- a/src/buildstream/_frontend/app.py
+++ b/src/buildstream/_frontend/app.py
@@ -446,15 +446,16 @@ class App:
     # if they are available in the execution context.
     #
     # Args:
-    #    element_name (str): The element's full name
-    #    element_key (tuple): The element's display key
+    #    element (Element): The element
     #
     # Returns:
     #    (str): The formatted prompt to display in the shell
     #
-    def shell_prompt(self, element_name, element_key):
+    def shell_prompt(self, element):
 
-        _, key, dim = element_key
+        element_name = element._get_full_name()
+
+        _, key, dim = element._get_display_key()
 
         if self.colors:
             prompt = (
@@ -703,10 +704,14 @@ class App:
                 if choice == "shell":
                     click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True)
                     try:
-                        unique_id, element_key = element
-                        prompt = self.shell_prompt(full_name, element_key)
+                        unique_id, _ = element
                         self.stream.shell(
-                            None, _Scope.BUILD, prompt, isolate=True, usebuildtree="always", unique_id=unique_id
+                            None,
+                            _Scope.BUILD,
+                            self.shell_prompt,
+                            isolate=True,
+                            usebuildtree="always",
+                            unique_id=unique_id,
                         )
                     except BstError as e:
                         click.echo("Error while attempting to create interactive shell: {}".format(e), err=True)
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 946024bb7..5e42bda68 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -578,12 +578,9 @@ def show(app, elements, deps, except_, order, format_):
     "--use-buildtree",
     "-t",
     "cli_buildtree",
-    type=click.Choice(["ask", "try", "always", "never"]),
-    default="ask",
-    show_default=True,
+    is_flag=True,
     help=(
-        "Stage a buildtree. If `always` is set, will always fail to "
-        "build if a buildtree is not available."
+        "Stage a buildtree. Will fail if a buildtree is not available."
         " --pull and pull-buildtrees configuration is needed "
         "if trying to query for remotely cached buildtrees."
     ),
@@ -617,11 +614,11 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command):
     from ..element import _Scope
     from .._project import HostMount
 
-    scope = _Scope.BUILD if build_ else _Scope.RUN
+    # Buildtree can only be used with build shells
+    if cli_buildtree:
+        build_ = True
 
-    # We may need to fetch dependency artifacts if we're pulling the artifact
-    selection = _PipelineSelection.ALL if pull_ else _PipelineSelection.NONE
-    use_buildtree = None
+    scope = _Scope.BUILD if build_ else _Scope.RUN
 
     with app.initialized():
         if not element:
@@ -629,104 +626,18 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command):
             if not element:
                 raise AppError('Missing argument "ELEMENT".')
 
-        elements = app.stream.load_selection((element,), selection=selection, use_artifact_config=True)
-
-        # last one will be the element we want to stage, previous ones are
-        # elements to try and pull
-        element = elements[-1]
-        pull_dependencies = elements[:-1] if pull_ else None
-
-        element_name = element._get_full_name()
-        element_key = element._get_display_key()
-
-        prompt = app.shell_prompt(element_name, element_key)
         mounts = [HostMount(path, host_path) for host_path, path in mount]
 
-        artifact_is_cached = element._cached()
-        buildtree_is_cached = element._cached_buildtree()
-        buildtree_exists = element._buildtree_exists()
-        can_attempt_pull = app.context.pull_buildtrees and pull_
-
-        if cli_buildtree in ("always", "try"):
-            if buildtree_is_cached:
-                use_buildtree = cli_buildtree
-            # If element is already cached, we can check the proto to see if the buildtree existed
-            elif artifact_is_cached:
-                if not buildtree_exists:
-                    if cli_buildtree == "always":
-                        # Exit early if it won't be possible to even fetch a buildtree with always option
-                        raise AppError("Artifact was created without buildtree, unable to launch shell with it")
-                    click.echo(
-                        "WARNING: Artifact created without buildtree, shell will be loaded without it", err=True
-                    )
-                elif can_attempt_pull:
-                    use_buildtree = cli_buildtree
-                    click.echo(
-                        "WARNING: buildtree is not cached locally but did exist, will attempt to pull from available remotes",
-                        err=True,
-                    )
-                else:
-                    if cli_buildtree == "always":
-                        # Exit early if it won't be possible to perform a fetch as pull semantics aren't present
-                        raise AppError(
-                            "Artifact has a buildtree but it isn't cached. Can be retried with --pull and pull-buildtrees configured"
-                        )
-                    click.echo("WARNING: buildtree is not cached locally, shell will be loaded without it", err=True)
-            # If element isn't cached at all, we can't check the proto to see if it existed so can't exit early
-            elif can_attempt_pull:
-                use_buildtree = cli_buildtree
-                if use_buildtree == "always":
-                    click.echo(
-                        "WARNING: Element is not cached so buildtree status unknown, will attempt to pull from available remotes",
-                        err=True,
-                    )
-            else:
-                if cli_buildtree == "always":
-                    # Exit early as there is no buildtree locally & can_attempt_pull is False
-                    raise AppError(
-                        "Artifact not cached locally. Can be retried with --pull and pull-buildtrees configured"
-                    )
-                click.echo("WARNING: buildtree is not cached locally, shell will be loaded without it", err=True)
-        else:
-            # If the value has defaulted to ask and in non interactive mode, don't consider the buildtree, this
-            # being the default behaviour of the command
-            if app.interactive and cli_buildtree == "ask":
-                if buildtree_is_cached and bool(click.confirm("Do you want to use the cached buildtree?")):
-                    use_buildtree = "always"
-                elif can_attempt_pull:
-                    # If buildtree not cached, check if it's worth presenting the user a dialogue
-                    message = None
-                    if artifact_is_cached:
-                        if buildtree_exists:
-                            message = "Buildtree not cached but can be pulled if in available remotes, do you want to use it?"
-                    else:
-                        message = "Element is not cached so buildtree status unknown, do you want to pull and use it?"
-                    if message:
-                        try:
-                            choice = click.prompt(
-                                message, type=click.Choice(["try", "always", "never"]), err=True, show_choices=True,
-                            )
-                        except click.Abort:
-                            click.echo("Aborting", err=True)
-                            sys.exit(-1)
-
-                        if choice != "never":
-                            use_buildtree = choice
-
-        # Raise warning if the element is cached in a failed state
-        if use_buildtree and element._cached_failure():
-            click.echo("WARNING: using a buildtree from a failed build.", err=True)
-
         try:
             exitcode = app.stream.shell(
                 element,
                 scope,
-                prompt,
+                app.shell_prompt,
                 mounts=mounts,
                 isolate=isolate,
                 command=command,
-                usebuildtree=use_buildtree,
-                pull_dependencies=pull_dependencies,
+                usebuildtree=cli_buildtree,
+                pull_=pull_,
             )
         except BstError as e:
             raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 3b0a308e7..af0537fe1 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -167,14 +167,14 @@ class Stream:
     # Run a shell
     #
     # Args:
-    #    element (Element): An Element object to run the shell for
+    #    element (str): The name of the element to run the shell for
     #    scope (_Scope): The scope for the shell (_Scope.BUILD or _Scope.RUN)
-    #    prompt (str): The prompt to display in the shell
+    #    prompt (function): A function to return the prompt to display in the shell
     #    mounts (list of HostMount): Additional directories to mount into the sandbox
     #    isolate (bool): Whether to isolate the environment like we do in builds
     #    command (list): An argv to launch in the sandbox, or None
-    #    usebuildtree (str): Whether to use a buildtree as the source, given cli option
-    #    pull_dependencies ([Element]|None): Elements to attempt to pull
+    #    usebuildtree (bool): Whether to use a buildtree as the source, given cli option
+    #    pull_ (bool): Whether to attempt to pull missing or incomplete artifacts
     #    unique_id: (str): Whether to use a unique_id to load an Element instance
     #
     # Returns:
@@ -189,61 +189,62 @@ class Stream:
         mounts=None,
         isolate=False,
         command=None,
-        usebuildtree=None,
-        pull_dependencies=None,
+        usebuildtree=False,
+        pull_=False,
         unique_id=None
     ):
 
         # Load the Element via the unique_id if given
         if unique_id and element is None:
             element = Plugin._lookup(unique_id)
+        else:
+            selection = _PipelineSelection.BUILD if scope == _Scope.BUILD else _PipelineSelection.RUN
+
+            elements = self.load_selection((element,), selection=selection, use_artifact_config=True)
+
+            # Get element to stage from `targets` list.
+            # If scope is BUILD, it will not be in the `elements` list.
+            assert len(self.targets) == 1
+            element = self.targets[0]
+            element._set_required(scope)
+
+            if pull_:
+                self._scheduler.clear_queues()
+                self._add_queue(PullQueue(self._scheduler))
+                plan = self._pipeline.add_elements([element], elements)
+                self._enqueue_plan(plan)
+                self._run()
 
         missing_deps = [dep for dep in self._pipeline.dependencies([element], scope) if not dep._cached()]
         if missing_deps:
-            if not pull_dependencies:
-                raise StreamError(
-                    "Elements need to be built or downloaded before staging a shell environment",
-                    detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
-                )
-            self._message(MessageType.INFO, "Attempting to fetch missing or incomplete artifacts")
-            self._scheduler.clear_queues()
-            self._add_queue(PullQueue(self._scheduler))
-            plan = self._pipeline.add_elements([element], missing_deps)
-            self._enqueue_plan(plan)
-            self._run()
+            raise StreamError(
+                "Elements need to be built or downloaded before staging a shell environment",
+                detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
+            )
 
-        buildtree = False
         # Check if we require a pull queue attempt, with given artifact state and context
         if usebuildtree:
             if not element._cached_buildtree():
-                require_buildtree = self._buildtree_pull_required([element])
-                # Attempt a pull queue for the given element if remote and context allow it
-                if require_buildtree:
-                    self._message(MessageType.INFO, "Attempting to fetch missing artifact buildtree")
-                    self._scheduler.clear_queues()
-                    self._add_queue(PullQueue(self._scheduler))
-                    self._enqueue_plan(require_buildtree)
-                    self._run()
-                    # Now check if the buildtree was successfully fetched
-                    if element._cached_buildtree():
-                        buildtree = True
-
-                if not buildtree:
-                    message = "Buildtree is not cached locally or in available remotes"
-                    if usebuildtree == "always":
-                        raise StreamError(message)
-
-                    self._message(MessageType.INFO, message + ", shell will be loaded without it")
-            else:
-                buildtree = True
+                remotes_message = " or in available remotes" if pull_ else ""
+                if not element._cached():
+                    message = "Artifact not cached locally" + remotes_message
+                elif element._buildtree_exists():
+                    message = "Buildtree is not cached locally" + remotes_message
+                else:
+                    message = "Artifact was created without buildtree"
+                raise StreamError(message)
+
+            # Raise warning if the element is cached in a failed state
+            if element._cached_failure():
+                self._message(MessageType.WARN, "using a buildtree from a failed build.")
 
         # Ensure we have our sources if we are launching a build shell
-        if scope == _Scope.BUILD and not buildtree:
+        if scope == _Scope.BUILD and not usebuildtree:
             self._fetch([element])
             self._pipeline.assert_sources_cached([element])
 
         return element._shell(
-            scope, mounts=mounts, isolate=isolate, prompt=prompt, command=command, usebuildtree=buildtree
+            scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree
         )
 
     # build()
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 3df8894f0..c9480d4fc 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1557,10 +1557,13 @@ class Element(Plugin):
 
     # _set_required():
     #
-    # Mark this element and its runtime dependencies as required.
+    # Mark this element and its dependencies as required.
     # This unblocks pull/fetch/build.
     #
-    def _set_required(self):
+    # Args:
+    #    scope (_Scope): The scope of dependencies to mark as required
+    #
+    def _set_required(self, scope=_Scope.RUN):
         assert utils._is_main_process(), "This has an impact on all elements and must be run in the main process"
 
         if self.__required:
@@ -1569,9 +1572,9 @@ class Element(Plugin):
 
         self.__required = True
 
-        # Request artifacts of runtime dependencies
-        for dep in self._dependencies(_Scope.RUN, recurse=False):
-            dep._set_required()
+        # Request artifacts of dependencies
+        for dep in self._dependencies(scope, recurse=False):
+            dep._set_required(scope=_Scope.RUN)
 
         # When an element becomes required, it must be assembled for
         # the current pipeline. `__schedule_assembly_when_necessary()`
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 0d80c1640..5164c0209 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -46,7 +46,7 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
     res.assert_success()
 
     res = cli_integration.run(
-        project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"]
+        project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "cat", "test"]
     )
     res.assert_success()
     assert "Hi" in res.output
@@ -67,53 +67,10 @@ def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
     res.assert_success()
 
     res = cli_integration.run(
-        project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"]
+        project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "cat", "test"]
     )
     res.assert_main_error(ErrorDomain.APP, None)
-    assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
-
-    # Now attempt the same with the try option, this should not attempt to find a buildtree
-    # and just launch the shell, however the cat should still fail.
-    res = cli_integration.run(
-        project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"]
-    )
-    assert "Artifact created without buildtree, shell will be loaded without it" in res.stderr
-    assert "Hi" not in res.output
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_staged_if_available(cli_integration, datafiles):
-    # Test that a build tree can be correctly detected.
-    project = str(datafiles)
-    element_name = "build-shell/buildtree.bst"
-
-    res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
-    res.assert_success()
-
-    res = cli_integration.run(
-        project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"]
-    )
-    res.assert_success()
-    assert "Hi" in res.output
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_staged_forced_false(cli_integration, datafiles):
-    # Test that if we ask not to have a build tree it is not there
-    project = str(datafiles)
-    element_name = "build-shell/buildtree.bst"
-
-    res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
-    res.assert_success()
-
-    res = cli_integration.run(
-        project=project, args=["shell", "--build", "--use-buildtree", "never", element_name, "--", "cat", "test"]
-    )
-    res.assert_shell_error()
-
-    assert "Hi" not in res.output
+    assert "Error launching shell: Artifact was created without buildtree" in res.stderr
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -128,10 +85,10 @@ def test_buildtree_from_failure(cli_integration, datafiles):
 
     # Assert that file has expected contents
     res = cli_integration.run(
-        project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+        project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]
     )
     res.assert_success()
-    assert "WARNING: using a buildtree from a failed build" in res.stderr
+    assert "WARNING using a buildtree from a failed build" in res.stderr
     assert "Hi" in res.output
 
 
@@ -150,10 +107,10 @@ def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles)
     res.assert_main_error(ErrorDomain.STREAM, None)
 
     res = cli_integration.run(
-        project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+        project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]
     )
     res.assert_main_error(ErrorDomain.APP, None)
-    assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr
+    assert "Error launching shell: Artifact was created without buildtree" in res.stderr
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -172,10 +129,10 @@ def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles
     res.assert_main_error(ErrorDomain.STREAM, None)
 
     res = cli_integration.run(
-        project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
+        project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]
     )
     res.assert_success()
-    assert "WARNING: using a buildtree from a failed build" in res.stderr
+    assert "WARNING using a buildtree from a failed build" in res.stderr
     assert "Hi" in res.output
 
 
@@ -206,9 +163,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
         result.assert_success()
 
         # Check it's using the cached build tree
-        res = cli.run(
-            project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
-        )
+        res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"])
         res.assert_success()
 
 
@@ -237,9 +192,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
         result.assert_success()
 
         # Check it's not using the cached build tree
-        res = cli.run(
-            project=project, args=["shell", "--build", element_name, "--use-buildtree", "never", "--", "cat", "test"]
-        )
+        res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"])
         res.assert_shell_error()
         assert "Hi" not in res.output
 
@@ -249,17 +202,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
         res.assert_shell_error()
         assert "Hi" not in res.output
 
-        # Check correctly handling the lack of buildtree, with 'try' not attempting to
-        # pull the buildtree as the user context is by default set to not pull them
-        # and --pull not given
-        res = cli.run(
-            project=project, args=["shell", "--build", element_name, "--use-buildtree", "try", "--", "cat", "test"]
-        )
-        assert "Hi" not in res.output
-        assert "Attempting to fetch missing artifact buildtrees" not in res.stderr
-        assert "WARNING: buildtree is not cached locally, shell will be loaded without it" in res.stderr
-
-        # Check correctly handling the lack of buildtree, with 'try' attempting and succeeding
+        # Check correctly handling the lack of buildtree, with '--use-buildtree' attempting and succeeding
         # to pull the buildtree as the user context allow the pulling of buildtrees and it is
         # available in the remote and --pull given
         res = cli.run(
@@ -271,35 +214,27 @@ def test_buildtree_options(cli, tmpdir, datafiles):
                 element_name,
                 "--pull",
                 "--use-buildtree",
-                "try",
                 "--",
                 "cat",
                 "test",
             ],
         )
-        assert "Attempting to fetch missing artifact buildtree" in res.stderr
         assert "Hi" in res.output
         shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "cas")))
         shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "artifacts")))
         assert cli.get_element_state(project, element_name) != "cached"
 
-        # Check it's not loading the shell at all with always set for the buildtree, when the
+        # Check it's not loading the shell at all with `--use-buildtree`, when the
         # user context does not allow for buildtree pulling and --pull is not given
         result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name])
         result.assert_success()
-        res = cli.run(
-            project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
-        )
+        res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"])
         res.assert_main_error(ErrorDomain.APP, None)
-        assert (
-            "Artifact has a buildtree but it isn't cached. Can be retried with --pull and pull-buildtrees configured"
-            in res.stderr
-        )
+        assert "Buildtree is not cached locally" in res.stderr
         assert "Hi" not in res.output
-        assert "Attempting to fetch missing artifact buildtree" not in res.stderr
 
         # Check that when user context is set to pull buildtrees and a remote has the buildtree,
-        # 'always' will attempt and succeed at pulling the missing buildtree with --pull set.
+        # '--use-buildtree' will attempt and succeed at pulling the missing buildtree with --pull set.
         res = cli.run(
             project=project,
             args=[
@@ -309,17 +244,13 @@ def test_buildtree_options(cli, tmpdir, datafiles):
                 element_name,
                 "--pull",
                 "--use-buildtree",
-                "always",
                 "--",
                 "cat",
                 "test",
             ],
         )
         assert "Hi" in res.output
-        assert (
-            "buildtree is not cached locally but did exist, will attempt to pull from available remotes" in res.stderr
-        )
-        assert "Attempting to fetch missing artifact buildtree" in res.stderr
+        assert res.get_pulled_elements() == [element_name]
 
 
 # Tests running pull and pull-buildtree options at the same time.
@@ -345,10 +276,10 @@ def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
         # and pull-buildtrees were not both set
         res = cli.run(
             project=project,
-            args=["shell", "--build", element_name, "--pull", "--use-buildtree", "always", "--", "cat", "test",],
+            args=["shell", "--build", element_name, "--pull", "--use-buildtree", "--", "cat", "test",],
         )
         res.assert_main_error(ErrorDomain.APP, None)
-        assert "Artifact not cached locally. Can be retried with --pull and pull-buildtrees configured" in res.stderr
+        assert "Buildtree is not cached locally" in res.stderr
 
         # Check it's using the cached build tree, because --pull
         # and pull-buildtrees were both set
@@ -361,7 +292,6 @@ def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
                 element_name,
                 "--pull",
                 "--use-buildtree",
-                "always",
                 "--",
                 "cat",
                 "test",
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index a2ea4841a..fab45b123 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -80,7 +80,7 @@ def test_workspace_commanddir(cli, datafiles):
     # using the cached buildtree.
     res = cli.run(
         project=project,
-        args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "find", "..", "-mindepth", "1",],
+        args=["shell", "--build", element_name, "--use-buildtree", "--", "find", "..", "-mindepth", "1",],
     )
     res.assert_success()
 
@@ -290,7 +290,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
     # the configure should have been run in the sandbox, so check the buildtree
     res = cli.run(
         project=project,
-        args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "find", ".", "-mindepth", "1",],
+        args=["shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1",],
     )
     res.assert_success()
 
@@ -311,7 +311,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
     assert not os.path.exists(os.path.join(workspace, "prepared-again"))
     res = cli.run(
         project=project,
-        args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "find", ".", "-mindepth", "1",],
+        args=["shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1",],
     )
     res.assert_success()
 
@@ -382,9 +382,7 @@ def test_workspace_failed_logs(cli, datafiles):
 
 
 def get_buildtree_file_contents(cli, project, element_name, filename):
-    res = cli.run(
-        project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", filename,],
-    )
+    res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", filename,],)
     res.assert_success()
     return res.output
 
diff --git a/tests/remoteexecution/buildtree.py b/tests/remoteexecution/buildtree.py
index 57e25cd14..317747fe8 100644
--- a/tests/remoteexecution/buildtree.py
+++ b/tests/remoteexecution/buildtree.py
@@ -63,8 +63,6 @@ def test_buildtree_remote(cli, tmpdir, datafiles):
         res.assert_success()
 
         # check it works this time
-        res = cli.run(
-            project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"]
-        )
+        res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"])
         res.assert_success()
         assert "Hi" in res.output
diff --git a/tests/remoteexecution/workspace.py b/tests/remoteexecution/workspace.py
index b525cefcd..cf8587230 100644
--- a/tests/remoteexecution/workspace.py
+++ b/tests/remoteexecution/workspace.py
@@ -115,7 +115,6 @@ def check_buildtree(
             "--build",
             element_name,
             "--use-buildtree",
-            "always",
             "--",
             "find",
             ".",
@@ -176,7 +175,7 @@ def check_buildtree(
 
 def get_timemark(cli, project, element_name, marker):
     result = cli.run(
-        project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", marker[1:]],
+        project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", marker[1:]],
     )
     result.assert_success()
     marker_time = int(result.output)