Merge branch 'tristan/fix-juncle-include-link' into 'master'

Fix including of files across linked junction boundaries

Closes #1359

See merge request BuildStream/buildstream!1993

18 files changed, 79 insertions, 3 deletions

diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 81e5fb032..a3af70f4b 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -318,7 +318,7 @@ class Loader:
             node.get_str_list(Symbol.ENV_NOCACHE, default=[]),
             node.get_mapping(Symbol.PUBLIC, default={}),
             node.get_mapping(Symbol.SANDBOX, default={}),
-            element_kind == "junction",
+            element_kind in ("junction", "link"),
         )
 
         # Cache it now, make sure it's already there before recursing
@@ -394,7 +394,7 @@ class Loader:
             raise
 
         kind = node.get_str(Symbol.KIND)
-        if kind == "junction":
+        if kind in ("junction", "link"):
             self._first_pass_options.process_node(node)
         else:
             self.project.ensure_fully_loaded()
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
index 97b0de242..1c1f6feb2 100644
--- a/src/buildstream/_loader/metaelement.py
+++ b/src/buildstream/_loader/metaelement.py
@@ -70,4 +70,4 @@ class MetaElement:
         self.dependencies = []
         self.strict_dependencies = []
         self.first_pass = first_pass
-        self.is_junction = kind == "junction"
+        self.is_junction = kind in ("junction", "link")
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 21dc2b9d2..b45c6c695 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -499,6 +499,20 @@ class Project:
         if self._fully_loaded:
             return
         assert self._partially_loaded
+
+        # Here we mark the project as fully loaded right away,
+        # before doing the work.
+        #
+        # This function will otherwise reenter itself infinitely:
+        #
+        #   * Ensuring the invariant that a parent project is fully
+        #     loaded before completing the load of this project, will
+        #     trigger this function when completing the load of subprojects.
+        #
+        #   * Completing the load of this project may include processing
+        #     some `(@)` include directives, which can directly trigger
+        #     the loading of subprojects.
+        #
         self._fully_loaded = True
 
         if self.junction:
diff --git a/tests/format/link.py b/tests/format/link.py
index e2c9e0b84..88b5569b0 100644
--- a/tests/format/link.py
+++ b/tests/format/link.py
@@ -5,6 +5,7 @@ import os
 
 import pytest
 
+from buildstream import _yaml
 from buildstream.testing import cli  # pylint: disable=unused-import
 from buildstream.exceptions import ErrorDomain, LoadErrorReason
 
@@ -149,3 +150,19 @@ def test_link_invalid_config(cli, tmpdir, datafiles, target, expected_error, exp
     project = os.path.join(str(datafiles), "invalid")
     result = cli.run(project=project, args=["show", target])
     result.assert_main_error(expected_error, expected_reason)
+
+
+#
+# Test including files across the boundry a link to a subproject's junction
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_cross_link_junction_include(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), "cross-link-junction-include")
+
+    # Show the variables and parse our test variable from the subsubproject
+    result = cli.run(project=project, args=["show", "--format", "%{vars}", "target.bst"])
+    result.assert_success()
+
+    # Read back some of our project defaults from the env
+    variables = _yaml.load_data(result.output)
+    assert variables.get_str("test") == "the test"
diff --git a/tests/format/link/cross-link-junction-include/project.conf b/tests/format/link/cross-link-junction-include/project.conf
new file mode 100644
index 000000000..27d789520
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/project.conf
@@ -0,0 +1,4 @@
+name: test
+min-version: 2.0
+
+(@): subsubproject.bst:file.yml
diff --git a/tests/format/link/cross-link-junction-include/subproject.bst b/tests/format/link/cross-link-junction-include/subproject.bst
new file mode 100644
index 000000000..c88189cb0
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject.bst
@@ -0,0 +1,4 @@
+kind: junction
+sources:
+- kind: local
+  path: subproject
diff --git a/tests/format/link/cross-link-junction-include/subproject/project.conf b/tests/format/link/cross-link-junction-include/subproject/project.conf
new file mode 100644
index 000000000..39a53e2ab
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/project.conf
@@ -0,0 +1,2 @@
+name: subtest
+min-version: 2.0
diff --git a/tests/format/link/cross-link-junction-include/subproject/sub.txt b/tests/format/link/cross-link-junction-include/subproject/sub.txt
new file mode 100644
index 000000000..f73f3093f
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/sub.txt
@@ -0,0 +1 @@
+file
diff --git a/tests/format/link/cross-link-junction-include/subproject/subsubproject.bst b/tests/format/link/cross-link-junction-include/subproject/subsubproject.bst
new file mode 100644
index 000000000..f535ab0e0
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subsubproject.bst
@@ -0,0 +1,4 @@
+kind: junction
+sources:
+- kind: local
+  path: subsubproject
diff --git a/tests/format/link/cross-link-junction-include/subproject/subsubproject/file.yml b/tests/format/link/cross-link-junction-include/subproject/subsubproject/file.yml
new file mode 100644
index 000000000..10d83ddd5
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subsubproject/file.yml
@@ -0,0 +1,2 @@
+variables:
+  test: the test
diff --git a/tests/format/link/cross-link-junction-include/subproject/subsubproject/project.conf b/tests/format/link/cross-link-junction-include/subproject/subsubproject/project.conf
new file mode 100644
index 000000000..d11bcbb30
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subsubproject/project.conf
@@ -0,0 +1,2 @@
+name: subsubtest
+min-version: 2.0
diff --git a/tests/format/link/cross-link-junction-include/subproject/subsubproject/subsub.txt b/tests/format/link/cross-link-junction-include/subproject/subsubproject/subsub.txt
new file mode 100644
index 000000000..f73f3093f
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subsubproject/subsub.txt
@@ -0,0 +1 @@
+file
diff --git a/tests/format/link/cross-link-junction-include/subproject/subsubproject/subtarget.bst b/tests/format/link/cross-link-junction-include/subproject/subsubproject/subtarget.bst
new file mode 100644
index 000000000..b6fea5b2a
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subsubproject/subtarget.bst
@@ -0,0 +1,4 @@
+kind: stack
+
+depends:
+- subsubsubproject.bst:target.bst
diff --git a/tests/format/link/cross-link-junction-include/subproject/subsubproject/target.bst b/tests/format/link/cross-link-junction-include/subproject/subsubproject/target.bst
new file mode 100644
index 000000000..afafac601
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subsubproject/target.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: subsub.txt
diff --git a/tests/format/link/cross-link-junction-include/subproject/subtarget.bst b/tests/format/link/cross-link-junction-include/subproject/subtarget.bst
new file mode 100644
index 000000000..c4549b373
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/subtarget.bst
@@ -0,0 +1,4 @@
+kind: stack
+
+depends:
+- subsubproject.bst:target.bst
diff --git a/tests/format/link/cross-link-junction-include/subproject/target.bst b/tests/format/link/cross-link-junction-include/subproject/target.bst
new file mode 100644
index 000000000..e24d9bbb4
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subproject/target.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: sub.txt
diff --git a/tests/format/link/cross-link-junction-include/subsubproject.bst b/tests/format/link/cross-link-junction-include/subsubproject.bst
new file mode 100644
index 000000000..5246bfea0
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/subsubproject.bst
@@ -0,0 +1,4 @@
+kind: link
+
+config:
+  target: subproject.bst:subsubproject.bst
diff --git a/tests/format/link/cross-link-junction-include/target.bst b/tests/format/link/cross-link-junction-include/target.bst
new file mode 100644
index 000000000..32bbc75bc
--- /dev/null
+++ b/tests/format/link/cross-link-junction-include/target.bst
@@ -0,0 +1,5 @@
+kind: stack
+
+depends:
+- subproject.bst:target.bst
+- subproject.bst:subtarget.bst