Handle cross junction elements in workspaces.

Workspaces are now index by colon separated junction path. This
now allows to create workspaces for elements in external projects.

Workspaces are owned by context instead of root project. However
it is initialized once top-level project is registered as we need
to resolve paths relatively to this top-level project.

Part of #359.

7 files changed, 154 insertions, 30 deletions

diff --git a/buildstream/_context.py b/buildstream/_context.py
index c0d49b245..114ac9e86 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -30,6 +30,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
 from ._message import Message, MessageType
 from ._profile import Topics, profile_start, profile_end
 from ._artifactcache import ArtifactCache
+from ._workspaces import Workspaces
 
 
 # Context()
@@ -113,6 +114,7 @@ class Context():
         self._message_depth = deque()
         self._projects = []
         self._project_overrides = {}
+        self._workspaces = None
 
     # load()
     #
@@ -219,6 +221,8 @@ class Context():
     #    project (Project): The project to add
     #
     def add_project(self, project):
+        if not self._projects:
+            self._workspaces = Workspaces(project)
         self._projects.append(project)
 
     # get_projects():
@@ -242,6 +246,9 @@ class Context():
     def get_toplevel_project(self):
         return self._projects[0]
 
+    def get_workspaces(self):
+        return self._workspaces
+
     # get_overrides():
     #
     # Fetch the override dictionary for the active project. This returns
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 143d59aa4..465124557 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -711,7 +711,7 @@ def workspace_close(app, remove_dir, all_, elements):
             sys.exit(0)
 
         if all_:
-            elements = [element_name for element_name, _ in app.project.workspaces.list()]
+            elements = [element_name for element_name, _ in app.context.get_workspaces().list()]
 
         elements = app.stream.redirect_element_names(elements)
 
@@ -763,7 +763,7 @@ def workspace_reset(app, soft, track_, all_, elements):
                 sys.exit(-1)
 
         if all_:
-            elements = tuple(element_name for element_name, _ in app.project.workspaces.list())
+            elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list())
 
         app.stream.workspace_reset(elements, soft=soft, track_first=track_)
 
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 25ffaf6d2..9f42bf613 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -34,7 +34,6 @@ from ._elementfactory import ElementFactory
 from ._sourcefactory import SourceFactory
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
-from ._workspaces import Workspaces
 
 
 # The separator we use for user specified aliases
@@ -87,7 +86,6 @@ class Project():
         self.refs = ProjectRefs(self.directory, 'project.refs')
         self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
 
-        self.workspaces = None                   # Workspaces
         self.options = None                      # OptionPool
         self.junction = junction                 # The junction Element object, if this is a subproject
         self.fail_on_overlap = False             # Whether overlaps are treated as errors
@@ -301,9 +299,6 @@ class Project():
         # Load artifacts pull/push configuration for this project
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config)
 
-        # Workspace configurations
-        self.workspaces = Workspaces(self)
-
         # Plugin origins and versions
         origins = _yaml.node_get(config, list, 'plugins', default_value=[])
         for origin in origins:
diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py
index 7c4ad6919..083822364 100644
--- a/buildstream/_scheduler/queue.py
+++ b/buildstream/_scheduler/queue.py
@@ -269,10 +269,11 @@ class Queue():
         # Handle any workspace modifications now
         #
         if job.workspace_dict:
-            project = element._get_project()
-            if project.workspaces.update_workspace(element.name, job.workspace_dict):
+            context = element._get_context()
+            workspaces = context.get_workspaces()
+            if workspaces.update_workspace(element._get_full_name(), job.workspace_dict):
                 try:
-                    project.workspaces.save_config()
+                    workspaces.save_config()
                 except BstError as e:
                     self._message(element, MessageType.ERROR, "Error saving workspaces", detail=str(e))
                 except Exception as e:   # pylint: disable=broad-except
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index f2806b4c8..c2fce58c0 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -434,8 +434,10 @@ class Stream():
             detail += "  \n".join(build_depends)
             raise StreamError("The given element has no sources", detail=detail)
 
+        workspaces = self._context.get_workspaces()
+
         # Check for workspace config
-        workspace = self._project.workspaces.get_workspace(target.name)
+        workspace = workspaces.get_workspace(target._get_full_name())
         if workspace:
             raise StreamError("Workspace '{}' is already defined at: {}"
                               .format(target.name, workspace.path))
@@ -460,13 +462,13 @@ class Stream():
         except OSError as e:
             raise StreamError("Failed to create workspace directory: {}".format(e)) from e
 
-        self._project.workspaces.create_workspace(target.name, workdir)
+        workspaces.create_workspace(target._get_full_name(), workdir)
 
         if not no_checkout:
             with target.timed_activity("Staging sources to {}".format(directory)):
                 target._open_workspace()
 
-        self._project.workspaces.save_config()
+        workspaces.save_config()
         self._message(MessageType.INFO, "Saved workspace configuration")
 
     # workspace_close
@@ -478,7 +480,8 @@ class Stream():
     #    remove_dir (bool): Whether to remove the associated directory
     #
     def workspace_close(self, element_name, *, remove_dir):
-        workspace = self._project.workspaces.get_workspace(element_name)
+        workspaces = self._context.get_workspaces()
+        workspace = workspaces.get_workspace(element_name)
 
         # Remove workspace directory if prompted
         if remove_dir:
@@ -491,8 +494,8 @@ class Stream():
                                       .format(workspace.path, e)) from e
 
         # Delete the workspace and save the configuration
-        self._project.workspaces.delete_workspace(element_name)
-        self._project.workspaces.save_config()
+        workspaces.delete_workspace(element_name)
+        workspaces.save_config()
         self._message(MessageType.INFO, "Closed workspace for {}".format(element_name))
 
     # workspace_reset
@@ -525,8 +528,10 @@ class Stream():
         if track_first:
             self._fetch(elements, track_elements=track_elements)
 
+        workspaces = self._context.get_workspaces()
+
         for element in elements:
-            workspace = self._project.workspaces.get_workspace(element.name)
+            workspace = workspaces.get_workspace(element._get_full_name())
 
             if soft:
                 workspace.prepared = False
@@ -542,15 +547,15 @@ class Stream():
                     raise StreamError("Could not remove  '{}': {}"
                                       .format(workspace.path, e)) from e
 
-            self._project.workspaces.delete_workspace(element.name)
-            self._project.workspaces.create_workspace(element.name, workspace.path)
+            workspaces.delete_workspace(element._get_full_name())
+            workspaces.create_workspace(element._get_full_name(), workspace.path)
 
             with element.timed_activity("Staging sources to {}".format(workspace.path)):
                 element._open_workspace()
 
             self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
 
-        self._project.workspaces.save_config()
+        workspaces.save_config()
 
     # workspace_exists
     #
@@ -566,11 +571,12 @@ class Stream():
     # True if there are any existing workspaces.
     #
     def workspace_exists(self, element_name=None):
+        workspaces = self._context.get_workspaces()
         if element_name:
-            workspace = self._project.workspaces.get_workspace(element_name)
+            workspace = workspaces.get_workspace(element_name)
             if workspace:
                 return True
-        elif any(self._project.workspaces.list()):
+        elif any(workspaces.list()):
             return True
 
         return False
@@ -581,7 +587,7 @@ class Stream():
     #
     def workspace_list(self):
         workspaces = []
-        for element_name, workspace_ in self._project.workspaces.list():
+        for element_name, workspace_ in self._context.get_workspaces().list():
             workspace_detail = {
                 'element': element_name,
                 'directory': workspace_.path,
diff --git a/buildstream/element.py b/buildstream/element.py
index 832f0dd93..796d79d66 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -673,7 +673,6 @@ class Element(Plugin):
         overlaps = OrderedDict()
         files_written = {}
         old_dep_keys = {}
-        project = self._get_project()
         workspace = self._get_workspace()
 
         if self.__can_build_incrementally() and workspace.last_successful:
@@ -702,7 +701,7 @@ class Element(Plugin):
                     # In case we are running `bst shell`, this happens in the
                     # main process and we need to update the workspace config
                     if utils._is_main_process():
-                        project.workspaces.save_config()
+                        self._get_context().get_workspaces().save_config()
 
             result = dep.stage_artifact(sandbox,
                                         path=path,
@@ -1393,12 +1392,11 @@ class Element(Plugin):
             # For this reason, it is safe to update and
             # save the workspaces configuration
             #
-            project = self._get_project()
             key = self._get_cache_key()
             workspace = self._get_workspace()
             workspace.last_successful = key
             workspace.clear_running_files()
-            project.workspaces.save_config()
+            self._get_context().get_workspaces().save_config()
 
     # _assemble():
     #
@@ -1763,8 +1761,8 @@ class Element(Plugin):
     #    (Workspace|None): A workspace associated with this element
     #
     def _get_workspace(self):
-        project = self._get_project()
-        return project.workspaces.get_workspace(self.name)
+        workspaces = self._get_context().get_workspaces()
+        return workspaces.get_workspace(self._get_full_name())
 
     # _write_script():
     #
@@ -1932,7 +1930,7 @@ class Element(Plugin):
                 'execution-environment': self.__sandbox_config.get_unique_key(),
                 'environment': cache_env,
                 'sources': [s._get_unique_key(workspace is None) for s in self.__sources],
-                'workspace': '' if workspace is None else workspace.get_key(),
+                'workspace': '' if workspace is None else workspace.get_key(self._get_project()),
                 'public': self.__public,
                 'cache': type(self.__artifacts).__name__
             }
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
new file mode 100644
index 000000000..eb2bc2eb8
--- /dev/null
+++ b/tests/frontend/cross_junction_workspace.py
@@ -0,0 +1,117 @@
+import os
+from tests.testutils import cli, create_repo
+from buildstream import _yaml
+
+
+def prepare_junction_project(cli, tmpdir):
+    main_project = tmpdir.join("main")
+    sub_project = tmpdir.join("sub")
+    os.makedirs(str(main_project))
+    os.makedirs(str(sub_project))
+
+    _yaml.dump({'name': 'main'}, str(main_project.join("project.conf")))
+    _yaml.dump({'name': 'sub'}, str(sub_project.join("project.conf")))
+
+    import_dir = tmpdir.join("import")
+    os.makedirs(str(import_dir))
+    with open(str(import_dir.join("hello.txt")), "w") as f:
+        f.write("hello!")
+
+    import_repo_dir = tmpdir.join("import_repo")
+    os.makedirs(str(import_repo_dir))
+    import_repo = create_repo("git", str(import_repo_dir))
+    import_ref = import_repo.create(str(import_dir))
+
+    _yaml.dump({'kind': 'import',
+                'sources': [import_repo.source_config(ref=import_ref)]},
+               str(sub_project.join("data.bst")))
+
+    sub_repo_dir = tmpdir.join("sub_repo")
+    os.makedirs(str(sub_repo_dir))
+    sub_repo = create_repo("git", str(sub_repo_dir))
+    sub_ref = sub_repo.create(str(sub_project))
+
+    _yaml.dump({'kind': 'junction',
+                'sources': [sub_repo.source_config(ref=sub_ref)]},
+               str(main_project.join("sub.bst")))
+
+    args = ['fetch', 'sub.bst']
+    result = cli.run(project=str(main_project), args=args)
+    result.assert_success()
+
+    return str(main_project)
+
+
+def open_cross_junction(cli, tmpdir):
+    project = prepare_junction_project(cli, tmpdir)
+    workspace = tmpdir.join("workspace")
+
+    element = 'sub.bst:data.bst'
+    args = ['workspace', 'open', element, str(workspace)]
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    assert cli.get_element_state(project, element) == 'buildable'
+    assert os.path.exists(str(workspace.join('hello.txt')))
+
+    return project, workspace
+
+
+def test_open_cross_junction(cli, tmpdir):
+    open_cross_junction(cli, tmpdir)
+
+
+def test_list_cross_junction(cli, tmpdir):
+    project, workspace = open_cross_junction(cli, tmpdir)
+
+    element = 'sub.bst:data.bst'
+
+    args = ['workspace', 'list']
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    loaded = _yaml.load_data(result.output)
+    assert isinstance(loaded.get('workspaces'), list)
+    workspaces = loaded['workspaces']
+    assert len(workspaces) == 1
+    assert 'element' in workspaces[0]
+    assert workspaces[0]['element'] == element
+
+
+def test_close_cross_junction(cli, tmpdir):
+    project, workspace = open_cross_junction(cli, tmpdir)
+
+    element = 'sub.bst:data.bst'
+    args = ['workspace', 'close', '--remove-dir', element]
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    assert not os.path.exists(str(workspace))
+
+    args = ['workspace', 'list']
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    loaded = _yaml.load_data(result.output)
+    assert isinstance(loaded.get('workspaces'), list)
+    workspaces = loaded['workspaces']
+    assert len(workspaces) == 0
+
+
+def test_close_all_cross_junction(cli, tmpdir):
+    project, workspace = open_cross_junction(cli, tmpdir)
+
+    args = ['workspace', 'close', '--remove-dir', '--all']
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    assert not os.path.exists(str(workspace))
+
+    args = ['workspace', 'list']
+    result = cli.run(project=project, args=args)
+    result.assert_success()
+
+    loaded = _yaml.load_data(result.output)
+    assert isinstance(loaded.get('workspaces'), list)
+    workspaces = loaded['workspaces']
+    assert len(workspaces) == 0