Allow tracking dependencies within sub-projects.valentindavid/359_cross_junction_elements

--track-cross-junctions now concerns crossing junctions rather than
forbidding elements in sub-project to be tracked.

Part of #359.

4 files changed, 196 insertions, 13 deletions

diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index ba27ca6b6..a280c22ee 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -346,6 +346,8 @@ class Pipeline():
     # lists targetted at tracking.
     #
     # Args:
+    #    project (Project): Project used for cross_junction filtering.
+    #                       All elements are expected to belong to that project.
     #    elements (list of Element): The list of elements to filter
     #    cross_junction_requested (bool): Whether the user requested
     #                                     cross junction tracking
@@ -353,12 +355,11 @@ class Pipeline():
     # Returns:
     #    (list of Element): The filtered or asserted result
     #
-    def track_cross_junction_filter(self, elements, cross_junction_requested):
+    def track_cross_junction_filter(self, project, elements, cross_junction_requested):
         # Filter out cross junctioned elements
-        if cross_junction_requested:
-            self._assert_junction_tracking(elements)
-        else:
-            elements = self._filter_cross_junctions(elements)
+        if not cross_junction_requested:
+            elements = self._filter_cross_junctions(project, elements)
+        self._assert_junction_tracking(elements)
 
         return elements
 
@@ -403,16 +404,17 @@ class Pipeline():
     # Filters out cross junction elements from the elements
     #
     # Args:
+    #    project (Project): The project on which elements are allowed
     #    elements (list of Element): The list of elements to be tracked
     #
     # Returns:
     #    (list): A filtered list of `elements` which does
     #            not contain any cross junction elements.
     #
-    def _filter_cross_junctions(self, elements):
+    def _filter_cross_junctions(self, project, elements):
         return [
             element for element in elements
-            if element._get_project() is self._project
+            if element._get_project() is project
         ]
 
     # _assert_junction_tracking()
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index c2fce58c0..80aac273e 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -831,12 +831,29 @@ class Stream():
         # done before resolving element states.
         #
         assert track_selection != PipelineSelection.PLAN
-        track_selected = self._pipeline.get_selection(track_elements, track_selection)
-        track_selected = self._pipeline.except_elements(track_elements,
-                                                        track_selected,
-                                                        track_except_elements)
-        track_selected = self._pipeline.track_cross_junction_filter(track_selected,
-                                                                    track_cross_junctions)
+
+        # Tracked elements are split by owner projects in order to
+        # filter cross junctions tracking dependencies on their
+        # respective project.
+        track_projects = {}
+        for element in track_elements:
+            project = element._get_project()
+            if project not in track_projects:
+                track_projects[project] = [element]
+            else:
+                track_projects[project].append(element)
+
+        track_selected = []
+
+        for project, project_elements in track_projects.items():
+            selected = self._pipeline.get_selection(project_elements, track_selection)
+            selected = self._pipeline.except_elements(project_elements,
+                                                      selected,
+                                                      track_except_elements)
+            selected = self._pipeline.track_cross_junction_filter(project,
+                                                                  selected,
+                                                                  track_cross_junctions)
+            track_selected.extend(selected)
 
         for element in track_selected:
             element._schedule_tracking()
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 2defc2349..51768d650 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -437,3 +437,46 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage):
 
     # Now assert element state (via bst show under the hood) of the dep again
     assert cli.get_element_state(project, 'junction-dep.bst') == 'waiting'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    subproject_path = os.path.join(project, 'files', 'sub-project')
+    junction_path = os.path.join(project, 'elements', 'junction.bst')
+    etc_files = os.path.join(subproject_path, 'files', 'etc-files')
+    repo_element_path = os.path.join(subproject_path, 'elements',
+                                     'import-etc-repo.bst')
+
+    configure_project(project, {
+        'ref-storage': ref_storage
+    })
+
+    repo = create_repo(kind, str(tmpdir.join('element_repo')))
+    ref = repo.create(etc_files)
+
+    generate_element(repo, repo_element_path)
+
+    generate_junction(str(tmpdir.join('junction_repo')),
+                      subproject_path, junction_path, store_ref=False)
+
+    # Track the junction itself first.
+    result = cli.run(project=project, args=['track', 'junction.bst'])
+    result.assert_success()
+
+    assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference'
+
+    # Track the cross junction element. -J is not given, it is implied.
+    result = cli.run(project=project, args=['track', 'junction.bst:import-etc-repo.bst'])
+
+    if ref_storage == 'inline':
+        # This is not allowed to track cross junction without project.refs.
+        result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources')
+    else:
+        result.assert_success()
+
+        assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable'
+
+        assert os.path.exists(os.path.join(project, 'project.refs'))
diff --git a/tests/frontend/track_cross_junction.py b/tests/frontend/track_cross_junction.py
new file mode 100644
index 000000000..089134ed3
--- /dev/null
+++ b/tests/frontend/track_cross_junction.py
@@ -0,0 +1,121 @@
+import os
+import pytest
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from buildstream import _yaml
+
+from . import generate_junction
+
+
+def generate_element(repo, element_path, dep_name=None):
+    element = {
+        'kind': 'import',
+        'sources': [
+            repo.source_config()
+        ]
+    }
+    if dep_name:
+        element['depends'] = [dep_name]
+
+    _yaml.dump(element, element_path)
+
+
+def generate_import_element(tmpdir, kind, project, name):
+    element_name = 'import-{}.bst'.format(name)
+    repo_element_path = os.path.join(project, 'elements', element_name)
+    files = str(tmpdir.join("imported_files_{}".format(name)))
+    os.makedirs(files)
+
+    with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f:
+        f.write(name)
+
+    subproject_path = os.path.join(str(tmpdir.join('sub-project-{}'.format(name))))
+
+    repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name))))
+    ref = repo.create(files)
+
+    generate_element(repo, repo_element_path)
+
+    return element_name
+
+
+def generate_project(tmpdir, name, config={}):
+    project_name = 'project-{}'.format(name)
+    subproject_path = os.path.join(str(tmpdir.join(project_name)))
+    os.makedirs(os.path.join(subproject_path, 'elements'))
+
+    project_conf = {
+        'name': name,
+        'element-path': 'elements'
+    }
+    project_conf.update(config)
+    _yaml.dump(project_conf, os.path.join(subproject_path, 'project.conf'))
+
+    return project_name, subproject_path
+
+
+def generate_simple_stack(project, name, dependencies):
+    element_name = '{}.bst'.format(name)
+    element_path = os.path.join(project, 'elements', element_name)
+    element = {
+        'kind': 'stack',
+        'depends': dependencies
+    }
+    _yaml.dump(element, element_path)
+
+    return element_name
+
+
+def generate_cross_element(project, subproject_name, import_name):
+    return generate_simple_stack(project, 'import-{}'.format(subproject_name),
+                                 [{
+                                     'junction': '{}.bst'.format(subproject_name),
+                                     'filename': import_name
+                                 }])
+
+
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_cross_junction_multiple_projects(cli, tmpdir, datafiles, kind):
+    tmpdir = tmpdir.join(kind)
+
+    # Generate 3 projects: main, a, b
+    _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
+    project_a, project_a_path = generate_project(tmpdir, 'a')
+    project_b, project_b_path = generate_project(tmpdir, 'b')
+
+    # Generate an element with a trackable source for each project
+    element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
+    element_b = generate_import_element(tmpdir, kind, project_b_path, 'b')
+    element_c = generate_import_element(tmpdir, kind, project, 'c')
+
+    # Create some indirections to the elements with dependencies to test --deps
+    stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a])
+    stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b])
+
+    # Create junctions for projects a and b in main.
+    junction_a = '{}.bst'.format(project_a)
+    junction_a_path = os.path.join(project, 'elements', junction_a)
+    generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
+
+    junction_b = '{}.bst'.format(project_b)
+    junction_b_path = os.path.join(project, 'elements', junction_b)
+    generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False)
+
+    # Track the junctions.
+    result = cli.run(project=project, args=['track', junction_a, junction_b])
+    result.assert_success()
+
+    # Import elements from a and b in to main.
+    imported_a = generate_cross_element(project, project_a, stack_a)
+    imported_b = generate_cross_element(project, project_b, stack_b)
+
+    # Generate a top level stack depending on everything
+    all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c])
+
+    # Track without following junctions. But explicitly also track the elements in project a.
+    result = cli.run(project=project, args=['track', '--deps', 'all', all_bst, '{}:{}'.format(junction_a, stack_a)])
+    result.assert_success()
+
+    # Elements in project b should not be tracked. But elements in project a and main should.
+    expected = [element_c,
+                '{}:{}'.format(junction_a, element_a)]
+    assert set(result.get_tracked_elements()) == set(expected)