Merge branch 'jonathan/workspace-fragment-create' into 'master'

Support invoking buildstream from a workspace outside a project

See merge request BuildStream/buildstream!924

11 files changed, 555 insertions, 56 deletions

diff --git a/NEWS b/NEWS
index b8003ceb6..d11e2c5c4 100644
--- a/NEWS
+++ b/NEWS
@@ -83,6 +83,10 @@ buildstream 1.3.1
     plugin has now a tag tracking feature instead. This can be enabled
     by setting 'track-tags'.
 
+  o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
+    commands to be run from a workspace that is not inside a project.
+
+
 =================
 buildstream 1.1.5
 =================
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 7ca60e7aa..5211fe54d 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -32,7 +32,7 @@ from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
 from ._artifactcache import ArtifactCache
 from ._artifactcache.cascache import CASCache
-from ._workspaces import Workspaces
+from ._workspaces import Workspaces, WorkspaceProjectCache
 from .plugin import _plugin_lookup
 
 
@@ -122,6 +122,10 @@ class Context():
         # remove a workspace directory.
         self.prompt_workspace_close_remove_dir = None
 
+        # Boolean, whether we double-check with the user that they meant to
+        # close the workspace when they're using it to access the project.
+        self.prompt_workspace_close_project_inaccessible = None
+
         # Boolean, whether we double-check with the user that they meant to do
         # a hard reset of a workspace, potentially losing changes.
         self.prompt_workspace_reset_hard = None
@@ -140,6 +144,7 @@ class Context():
         self._projects = []
         self._project_overrides = {}
         self._workspaces = None
+        self._workspace_project_cache = WorkspaceProjectCache()
         self._log_handle = None
         self._log_filename = None
         self._cascache = None
@@ -250,12 +255,15 @@ class Context():
             defaults, Mapping, 'prompt')
         _yaml.node_validate(prompt, [
             'auto-init', 'really-workspace-close-remove-dir',
+            'really-workspace-close-project-inaccessible',
             'really-workspace-reset-hard',
         ])
         self.prompt_auto_init = _node_get_option_str(
             prompt, 'auto-init', ['ask', 'no']) == 'ask'
         self.prompt_workspace_close_remove_dir = _node_get_option_str(
             prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
+        self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
+            prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
         self.prompt_workspace_reset_hard = _node_get_option_str(
             prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
 
@@ -285,7 +293,7 @@ class Context():
     #
     def add_project(self, project):
         if not self._projects:
-            self._workspaces = Workspaces(project)
+            self._workspaces = Workspaces(project, self._workspace_project_cache)
         self._projects.append(project)
 
     # get_projects():
@@ -312,6 +320,16 @@ class Context():
     def get_workspaces(self):
         return self._workspaces
 
+    # get_workspace_project_cache():
+    #
+    # Return the WorkspaceProjectCache object used for this BuildStream invocation
+    #
+    # Returns:
+    #    (WorkspaceProjectCache): The WorkspaceProjectCache object
+    #
+    def get_workspace_project_cache(self):
+        return self._workspace_project_cache
+
     # get_overrides():
     #
     # Fetch the override dictionary for the active project. This returns
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index ef0dadb13..e1fab38a0 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -59,18 +59,9 @@ def complete_target(args, incomplete):
     :return: all the possible user-specified completions for the param
     """
 
+    from .. import utils
     project_conf = 'project.conf'
 
-    def ensure_project_dir(directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, project_conf)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
-                break
-            directory = parent_dir
-
-        return directory
-
     # First resolve the directory, in case there is an
     # active --directory/-C option
     #
@@ -89,7 +80,7 @@ def complete_target(args, incomplete):
     else:
         # Check if this directory or any of its parent directories
         # contain a project config file
-        base_directory = ensure_project_dir(base_directory)
+        base_directory, _ = utils._search_upward_for_files(base_directory, [project_conf])
 
     # Now parse the project.conf just to find the element path,
     # this is unfortunately a bit heavy.
@@ -772,11 +763,19 @@ def workspace_close(app, remove_dir, all_, elements):
 
         elements = app.stream.redirect_element_names(elements)
 
-        # Check that the workspaces in question exist
+        # Check that the workspaces in question exist, and that it's safe to
+        # remove them.
         nonexisting = []
         for element_name in elements:
             if not app.stream.workspace_exists(element_name):
                 nonexisting.append(element_name)
+            if (app.stream.workspace_is_required(element_name) and app.interactive and
+                    app.context.prompt_workspace_close_project_inaccessible):
+                click.echo("Removing '{}' will prevent you from running "
+                           "BuildStream commands from the current directory".format(element_name))
+                if not click.confirm('Are you sure you want to close this workspace?'):
+                    click.echo('Aborting', err=True)
+                    sys.exit(-1)
         if nonexisting:
             raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
 
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 386036488..90ed58e14 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -41,6 +41,7 @@ from .element import Element
 from ._message import Message, MessageType
 from ._includes import Includes
 from ._platform import Platform
+from ._workspaces import WORKSPACE_PROJECT_FILE
 
 
 # Project Configuration file
@@ -95,8 +96,10 @@ class Project():
         # The project name
         self.name = None
 
-        # The project directory
-        self.directory = self._ensure_project_dir(directory)
+        self._context = context  # The invocation Context, a private member
+
+        # The project directory, and whether the element whose workspace it was invoked from
+        self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
 
         # Absolute path to where elements are loaded from within the project
         self.element_path = None
@@ -117,7 +120,6 @@ class Project():
         #
         # Private Members
         #
-        self._context = context  # The invocation Context
 
         self._default_mirror = default_mirror    # The name of the preferred mirror.
 
@@ -371,6 +373,14 @@ class Project():
 
         self._load_second_pass()
 
+    # invoked_from_workspace_element()
+    #
+    # Returns the element whose workspace was used to invoke buildstream
+    # if buildstream was invoked from an external workspace
+    #
+    def invoked_from_workspace_element(self):
+        return self._invoked_from_workspace_element
+
     # cleanup()
     #
     # Cleans up resources used loading elements
@@ -650,7 +660,7 @@ class Project():
         # Source url aliases
         output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
 
-    # _ensure_project_dir()
+    # _find_project_dir()
     #
     # Returns path of the project directory, if a configuration file is found
     # in given directory or any of its parent directories.
@@ -661,18 +671,30 @@ class Project():
     # Raises:
     #    LoadError if project.conf is not found
     #
-    def _ensure_project_dir(self, directory):
-        directory = os.path.abspath(directory)
-        while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
-            parent_dir = os.path.dirname(directory)
-            if directory == parent_dir:
-                raise LoadError(
-                    LoadErrorReason.MISSING_PROJECT_CONF,
-                    '{} not found in current directory or any of its parent directories'
-                    .format(_PROJECT_CONF_FILE))
-            directory = parent_dir
+    # Returns:
+    #    (str) - the directory that contains the project, and
+    #    (str) - the name of the element required to find the project, or None
+    #
+    def _find_project_dir(self, directory):
+        workspace_element = None
+        found_directory, filename = utils._search_upward_for_files(
+            directory, [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE]
+        )
+        if filename == _PROJECT_CONF_FILE:
+            project_directory = found_directory
+        elif filename == WORKSPACE_PROJECT_FILE:
+            workspace_project_cache = self._context.get_workspace_project_cache()
+            workspace_project = workspace_project_cache.get(found_directory)
+            if workspace_project:
+                project_directory = workspace_project.get_default_project_path()
+                workspace_element = workspace_project.get_default_element()
+        else:
+            raise LoadError(
+                LoadErrorReason.MISSING_PROJECT_CONF,
+                '{} not found in current directory or any of its parent directories'
+                .format(_PROJECT_CONF_FILE))
 
-        return directory
+        return project_directory, workspace_element
 
     def _load_plugin_factories(self, config, output):
         plugin_source_origins = []   # Origins of custom sources
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index 6f298c259..6d4af1e79 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -581,15 +581,7 @@ class Stream():
                     todo_elements = "\nDid not try to create workspaces for " + todo_elements
                 raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
 
-            workspaces.create_workspace(target._get_full_name(), directory)
-
-            if not no_checkout:
-                with target.timed_activity("Staging sources to {}".format(directory)):
-                    target._open_workspace()
-
-            # Saving the workspace once it is set up means that if the next workspace fails to be created before
-            # the configuration gets saved. The successfully created workspace still gets saved.
-            workspaces.save_config()
+            workspaces.create_workspace(target, directory, checkout=not no_checkout)
             self._message(MessageType.INFO, "Created a workspace for element: {}"
                           .format(target._get_full_name()))
 
@@ -672,10 +664,7 @@ class Stream():
                                       .format(workspace_path, e)) from e
 
             workspaces.delete_workspace(element._get_full_name())
-            workspaces.create_workspace(element._get_full_name(), workspace_path)
-
-            with element.timed_activity("Staging sources to {}".format(workspace_path)):
-                element._open_workspace()
+            workspaces.create_workspace(element, workspace_path, checkout=True)
 
             self._message(MessageType.INFO,
                           "Reset workspace for {} at: {}".format(element.name,
@@ -707,6 +696,20 @@ class Stream():
 
         return False
 
+    # workspace_is_required()
+    #
+    # Checks whether the workspace belonging to element_name is required to
+    # load the project
+    #
+    # Args:
+    #    element_name (str): The element whose workspace may be required
+    #
+    # Returns:
+    #    (bool): True if the workspace is required
+    def workspace_is_required(self, element_name):
+        invoked_elm = self._project.invoked_from_workspace_element()
+        return invoked_elm == element_name
+
     # workspace_list
     #
     # Serializes the workspaces and dumps them in YAML to stdout.
diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py
index 468073f05..24a3cc8d3 100644
--- a/buildstream/_workspaces.py
+++ b/buildstream/_workspaces.py
@@ -25,6 +25,202 @@ from ._exceptions import LoadError, LoadErrorReason
 
 
 BST_WORKSPACE_FORMAT_VERSION = 3
+BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
+WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
+
+
+# WorkspaceProject()
+#
+# An object to contain various helper functions and data required for
+# referring from a workspace back to buildstream.
+#
+# Args:
+#    directory (str): The directory that the workspace exists in.
+#
+class WorkspaceProject():
+    def __init__(self, directory):
+        self._projects = []
+        self._directory = directory
+
+    # get_default_project_path()
+    #
+    # Retrieves the default path to a project.
+    #
+    # Returns:
+    #    (str): The path to a project
+    #
+    def get_default_project_path(self):
+        return self._projects[0]['project-path']
+
+    # get_default_element()
+    #
+    # Retrieves the name of the element that owns this workspace.
+    #
+    # Returns:
+    #    (str): The name of an element
+    #
+    def get_default_element(self):
+        return self._projects[0]['element-name']
+
+    # to_dict()
+    #
+    # Turn the members data into a dict for serialization purposes
+    #
+    # Returns:
+    #    (dict): A dict representation of the WorkspaceProject
+    #
+    def to_dict(self):
+        ret = {
+            'projects': self._projects,
+            'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION,
+        }
+        return ret
+
+    # from_dict()
+    #
+    # Loads a new WorkspaceProject from a simple dictionary
+    #
+    # Args:
+    #    directory (str): The directory that the workspace exists in
+    #    dictionary (dict): The dict to generate a WorkspaceProject from
+    #
+    # Returns:
+    #   (WorkspaceProject): A newly instantiated WorkspaceProject
+    #
+    @classmethod
+    def from_dict(cls, directory, dictionary):
+        # Only know how to handle one format-version at the moment.
+        format_version = int(dictionary['format-version'])
+        assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \
+            "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary)
+
+        workspace_project = cls(directory)
+        for item in dictionary['projects']:
+            workspace_project.add_project(item['project-path'], item['element-name'])
+
+        return workspace_project
+
+    # load()
+    #
+    # Loads the WorkspaceProject for a given directory.
+    #
+    # Args:
+    #    directory (str): The directory
+    # Returns:
+    #    (WorkspaceProject): The created WorkspaceProject, if in a workspace, or
+    #    (NoneType): None, if the directory is not inside a workspace.
+    #
+    @classmethod
+    def load(cls, directory):
+        workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE)
+        if os.path.exists(workspace_file):
+            data_dict = _yaml.load(workspace_file)
+            return cls.from_dict(directory, data_dict)
+        else:
+            return None
+
+    # write()
+    #
+    # Writes the WorkspaceProject to disk
+    #
+    def write(self):
+        os.makedirs(self._directory, exist_ok=True)
+        _yaml.dump(self.to_dict(), self.get_filename())
+
+    # get_filename()
+    #
+    # Returns the full path to the workspace local project file
+    #
+    def get_filename(self):
+        return os.path.join(self._directory, WORKSPACE_PROJECT_FILE)
+
+    # add_project()
+    #
+    # Adds an entry containing the project's path and element's name.
+    #
+    # Args:
+    #    project_path (str): The path to the project that opened the workspace.
+    #    element_name (str): The name of the element that the workspace belongs to.
+    #
+    def add_project(self, project_path, element_name):
+        assert (project_path and element_name)
+        self._projects.append({'project-path': project_path, 'element-name': element_name})
+
+
+# WorkspaceProjectCache()
+#
+# A class to manage workspace project data for multiple workspaces.
+#
+class WorkspaceProjectCache():
+    def __init__(self):
+        self._projects = {}  # Mapping of a workspace directory to its WorkspaceProject
+
+    # get()
+    #
+    # Returns a WorkspaceProject for a given directory, retrieving from the cache if
+    # present.
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #
+    # Returns:
+    #    (WorkspaceProject): The WorkspaceProject that was found for that directory.
+    #    or      (NoneType): None, if no WorkspaceProject can be found.
+    #
+    def get(self, directory):
+        try:
+            workspace_project = self._projects[directory]
+        except KeyError:
+            workspace_project = WorkspaceProject.load(directory)
+            if workspace_project:
+                self._projects[directory] = workspace_project
+
+        return workspace_project
+
+    # add()
+    #
+    # Adds the project path and element name to the WorkspaceProject that exists
+    # for that directory
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #    project_path (str): The path to the project that refers to this workspace
+    #    element_name (str): The element in the project that was refers to this workspace
+    #
+    # Returns:
+    #    (WorkspaceProject): The WorkspaceProject that was found for that directory.
+    #
+    def add(self, directory, project_path, element_name):
+        workspace_project = self.get(directory)
+        if not workspace_project:
+            workspace_project = WorkspaceProject(directory)
+            self._projects[directory] = workspace_project
+
+        workspace_project.add_project(project_path, element_name)
+        return workspace_project
+
+    # remove()
+    #
+    # Removes the project path and element name from the WorkspaceProject that exists
+    # for that directory.
+    #
+    # NOTE: This currently just deletes the file, but with support for multiple
+    # projects opening the same workspace, this will involve decreasing the count
+    # and deleting the file if there are no more projects.
+    #
+    # Args:
+    #    directory (str): The directory to search for a WorkspaceProject.
+    #
+    def remove(self, directory):
+        workspace_project = self.get(directory)
+        if not workspace_project:
+            raise LoadError(LoadErrorReason.MISSING_FILE,
+                            "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE))
+        path = workspace_project.get_filename()
+        try:
+            os.unlink(path)
+        except FileNotFoundError:
+            pass
 
 
 # Workspace()
@@ -174,10 +370,15 @@ class Workspace():
         if recalculate or self._key is None:
             fullpath = self.get_absolute_path()
 
+            excluded_files = (WORKSPACE_PROJECT_FILE,)
+
             # Get a list of tuples of the the project relative paths and fullpaths
             if os.path.isdir(fullpath):
                 filelist = utils.list_relative_paths(fullpath)
-                filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
+                filelist = [
+                    (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
+                    if relpath not in excluded_files
+                ]
             else:
                 filelist = [(self.get_absolute_path(), fullpath)]
 
@@ -199,12 +400,14 @@ class Workspace():
 #
 # Args:
 #    toplevel_project (Project): Top project used to resolve paths.
+#    workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects
 #
 class Workspaces():
-    def __init__(self, toplevel_project):
+    def __init__(self, toplevel_project, workspace_project_cache):
         self._toplevel_project = toplevel_project
         self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
         self._workspaces = self._load_config()
+        self._workspace_project_cache = workspace_project_cache
 
     # list()
     #
@@ -219,19 +422,36 @@ class Workspaces():
 
     # create_workspace()
     #
-    # Create a workspace in the given path for the given element.
+    # Create a workspace in the given path for the given element, and potentially
+    # checks-out the target into it.
     #
     # Args:
-    #    element_name (str) - The element name to create a workspace for
+    #    target (Element) - The element to create a workspace for
     #    path (str) - The path in which the workspace should be kept
+    #    checkout (bool): Whether to check-out the element's sources into the directory
     #
-    def create_workspace(self, element_name, path):
-        if path.startswith(self._toplevel_project.directory):
-            path = os.path.relpath(path, self._toplevel_project.directory)
+    def create_workspace(self, target, path, *, checkout):
+        element_name = target._get_full_name()
+        project_dir = self._toplevel_project.directory
+        if path.startswith(project_dir):
+            workspace_path = os.path.relpath(path, project_dir)
+        else:
+            workspace_path = path
 
-        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
+        self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
 
-        return self._workspaces[element_name]
+        if checkout:
+            with target.timed_activity("Staging sources to {}".format(path)):
+                target._open_workspace()
+
+        workspace_project = self._workspace_project_cache.add(path, project_dir, element_name)
+        project_file_path = workspace_project.get_filename()
+
+        if os.path.exists(project_file_path):
+            target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE))
+        workspace_project.write()
+
+        self.save_config()
 
     # get_workspace()
     #
@@ -280,8 +500,19 @@ class Workspaces():
     #    element_name (str) - The element name whose workspace to delete
     #
     def delete_workspace(self, element_name):
+        workspace = self.get_workspace(element_name)
         del self._workspaces[element_name]
 
+        # Remove from the cache if it exists
+        try:
+            self._workspace_project_cache.remove(workspace.get_absolute_path())
+        except LoadError as e:
+            # We might be closing a workspace with a deleted directory
+            if e.reason == LoadErrorReason.MISSING_FILE:
+                pass
+            else:
+                raise
+
     # save_config()
     #
     # Dump the current workspace element to the project configuration
diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml
index e81961c53..4429d1f6d 100644
--- a/buildstream/data/userconfig.yaml
+++ b/buildstream/data/userconfig.yaml
@@ -128,6 +128,14 @@ prompt:
   #
   really-workspace-close-remove-dir: ask
 
+  # Whether to really proceed with 'bst workspace close' when doing so would
+  # stop them from running bst commands in this workspace.
+  #
+  #  ask - Ask the user if they are sure.
+  #  yes - Always close, without asking.
+  #
+  really-workspace-close-project-inaccessible: ask
+
   # Whether to really proceed with 'bst workspace reset' doing a hard reset of
   # a workspace, potentially losing changes.
   #
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 2fe9ab7dc..4da29c259 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -1259,3 +1259,34 @@ def _message_digest(message_buffer):
     digest.hash = sha.hexdigest()
     digest.size_bytes = len(message_buffer)
     return digest
+
+
+# _search_upward_for_files()
+#
+# Searches upwards (from directory, then directory's parent directory...)
+# for any of the files listed in `filenames`.
+#
+# If multiple filenames are specified, and present in the same directory,
+# the first filename in the list will be returned.
+#
+# Args:
+#    directory (str): The directory to begin searching for files from
+#    filenames (list of str): The names of files to search for
+#
+# Returns:
+#    (str): The directory a file was found in, or None
+#    (str): The name of the first file that was found in that directory, or None
+#
+def _search_upward_for_files(directory, filenames):
+    directory = os.path.abspath(directory)
+    while True:
+        for filename in filenames:
+            file_path = os.path.join(directory, filename)
+            if os.path.isfile(file_path):
+                return directory, filename
+
+        parent_dir = os.path.dirname(directory)
+        if directory == parent_dir:
+            # i.e. we've reached the root of the filesystem
+            return None, None
+        directory = parent_dir
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index fb2b34c43..ad2a62626 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -115,3 +115,23 @@ def test_close_all_cross_junction(cli, tmpdir):
     assert isinstance(loaded.get('workspaces'), list)
     workspaces = loaded['workspaces']
     assert len(workspaces) == 0
+
+
+def test_subdir_command_cross_junction(cli, tmpdir):
+    # i.e. commands can be run successfully from a subdirectory of the
+    # junction's workspace, in case project loading logic has gone wrong
+    project = prepare_junction_project(cli, tmpdir)
+    workspace = os.path.join(str(tmpdir), 'workspace')
+    junction_element = 'sub.bst'
+
+    # Open the junction as a workspace
+    args = ['workspace', 'open', '--directory', workspace, junction_element]
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    # Run commands from a subdirectory of the workspace
+    newdir = os.path.join(str(workspace), "newdir")
+    element_name = 'data.bst'
+    os.makedirs(newdir)
+    result = cli.run(project=str(workspace), args=['-C', newdir, 'show', element_name])
+    result.assert_success()
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index bc928ad51..baca869bf 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -31,6 +31,7 @@ import shutil
 import subprocess
 from ruamel.yaml.comments import CommentedSet
 from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
+from tests.testutils import create_artifact_share
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("kind", repo_kinds)
 @pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
-def test_build(cli, tmpdir, datafiles, kind, strict):
+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
+    tmpdir = tmpdir_factory.mktemp('')
     element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
     checkout = os.path.join(str(tmpdir), 'checkout')
+    args_pre = ['-C', workspace] if call_from == "workspace" else []
 
     # Modify workspace
     shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
@@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
     # Build modified workspace
     assert cli.get_element_state(project, element_name) == 'buildable'
     assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
-    result = cli.run(project=project, args=['build', element_name])
+    result = cli.run(project=project, args=args_pre + ['build', element_name])
     result.assert_success()
     assert cli.get_element_state(project, element_name) == 'cached'
     assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 
     # Checkout the result
-    result = cli.run(project=project, args=[
-        'checkout', element_name, checkout
-    ])
+    result = cli.run(project=project,
+                     args=args_pre + ['checkout', element_name, checkout])
     result.assert_success()
 
     # Check that the pony.conf from the modified workspace exists
@@ -1055,3 +1058,137 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
         result = cli.run(project=project, args=["build", element_name])
         assert "BUG" not in result.stderr
         assert cli.get_element_state(project, element_name) != "cached"
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize('subdir', [True, False], ids=["subdir", "no-subdir"])
+def test_external_fetch(cli, datafiles, tmpdir_factory, subdir):
+    # Fetching from a workspace outside a project doesn't fail horribly
+    tmpdir = tmpdir_factory.mktemp('')
+    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    if subdir:
+        call_dir = os.path.join(workspace, 'usr')
+    else:
+        call_dir = workspace
+
+    result = cli.run(project=project, args=['-C', call_dir, 'fetch', element_name])
+    result.assert_success()
+
+    # We already fetched it by opening the workspace, but we're also checking
+    # `bst show` works here
+    assert cli.get_element_state(project, element_name) == 'buildable'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_push_pull(cli, datafiles, tmpdir_factory):
+    # Pushing and pulling to/from an artifact cache works from an external workspace
+    tmpdir = tmpdir_factory.mktemp('')
+    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
+        result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
+        result.assert_success()
+
+        cli.configure({
+            'artifacts': {'url': share.repo, 'push': True}
+        })
+
+        result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
+        result.assert_success()
+
+        result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
+        result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_track(cli, datafiles, tmpdir_factory):
+    # Tracking does not get horribly confused
+    tmpdir = tmpdir_factory.mktemp('')
+    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
+
+    # The workspace is necessarily already tracked, so we only care that
+    # there's no weird errors.
+    result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_open_other(cli, datafiles, tmpdir_factory):
+    # From inside an external workspace, open another workspace
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    # Closing the other element first, because I'm too lazy to create an
+    # element without opening it
+    result = cli.run(project=project, args=['workspace', 'close', beta_element])
+    result.assert_success()
+
+    result = cli.run(project=project, args=[
+        '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
+    ])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_close_other(cli, datafiles, tmpdir_factory):
+    # From inside an external workspace, close the other workspace
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_close_self(cli, datafiles, tmpdir_factory):
+    # From inside an external workspace, close it
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_reset_other(cli, datafiles, tmpdir_factory):
+    tmpdir1 = tmpdir_factory.mktemp('')
+    tmpdir2 = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
+    beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
+
+    result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_reset_self(cli, datafiles, tmpdir):
+    element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    # Command succeeds
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
+    result.assert_success()
+
+    # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_external_list(cli, datafiles, tmpdir_factory):
+    tmpdir = tmpdir_factory.mktemp('')
+    # Making use of the assumption that it's the same project in both invocations of open_workspace
+    element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
+
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
+    result.assert_success()
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 68535bfdc..1cec21818 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles):
 
     result = execute_shell(cli, project, ["true"], element=element_name)
     assert result.exit_code == 0
+
+
+# Test that a shell can be opened from an external workspace
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
+def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
+    tmpdir = tmpdir_factory.mktemp("")
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_name = 'autotools/amhello.bst'
+    workspace_dir = os.path.join(str(tmpdir), 'workspace')
+
+    result = cli.run(project=project, args=[
+        'workspace', 'open', '--directory', workspace_dir, element_name
+    ])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
+    result.assert_success()
+
+    command = ['shell']
+    if build_shell == 'build':
+        command.append('--build')
+    command.extend([element_name, '--', 'true'])
+    result = cli.run(project=project, cwd=workspace_dir, args=command)
+    result.assert_success()