Merge branch 'bschubert/allow-source-variables-access' into 'master'

Resolve source's config with the element's variable

Closes #1127

See merge request BuildStream/buildstream!1924

12 files changed, 78 insertions, 21 deletions

diff --git a/NEWS b/NEWS
index af362eb2f..10563b127 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,7 @@ Format
   o BREAKING CHANGE: Now deprecation warnings are suppressed using the `allow-deprecated`
     configuration with the plugin origins in project.conf, instead of on the
     source/element overrides section (See issue #1291)
+  o Variables from an element can now be used in source configurations
 
 
 Plugins
diff --git a/src/buildstream/_pluginfactory/sourcefactory.py b/src/buildstream/_pluginfactory/sourcefactory.py
index 5277577d4..d616702ef 100644
--- a/src/buildstream/_pluginfactory/sourcefactory.py
+++ b/src/buildstream/_pluginfactory/sourcefactory.py
@@ -45,6 +45,7 @@ class SourceFactory(PluginFactory):
     #    context (object): The Context object for processing
     #    project (object): The project object
     #    meta (object): The loaded MetaSource
+    #    variables (Variables): The variables available to the source
     #
     # Returns:
     #    A newly created Source object of the appropriate kind
@@ -53,7 +54,7 @@ class SourceFactory(PluginFactory):
     #    PluginError (if the kind lookup failed)
     #    LoadError (if the source itself took issue with the config)
     #
-    def create(self, context, project, meta):
+    def create(self, context, project, meta, variables):
         source_type, _ = self.lookup(context.messenger, meta.kind, meta.provenance)
-        source = source_type(context, project, meta)
+        source = source_type(context, project, meta, variables)
         return source
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 06c677051..1c6e7e950 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -375,16 +375,17 @@ class Project:
     #
     # Args:
     #    meta (MetaSource): The loaded MetaSource
+    #    variables (Variables): The list of variables available to the source
     #    first_pass (bool): Whether to use first pass configuration (for junctions)
     #
     # Returns:
     #    (Source): A newly created Source object of the appropriate kind
     #
-    def create_source(self, meta, *, first_pass=False):
+    def create_source(self, meta, variables, *, first_pass=False):
         if first_pass:
-            return self.first_pass_config.source_factory.create(self._context, self, meta)
+            return self.first_pass_config.source_factory.create(self._context, self, meta, variables)
         else:
-            return self.config.source_factory.create(self._context, self, meta)
+            return self.config.source_factory.create(self._context, self, meta, variables)
 
     # get_alias_uri()
     #
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 67a915e48..fa39eba12 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -901,7 +901,7 @@ class Element(Plugin):
         # Instantiate sources and generate their keys
         for meta_source in meta.sources:
             meta_source.first_pass = meta.is_junction
-            source = meta.project.create_source(meta_source, first_pass=meta.first_pass)
+            source = meta.project.create_source(meta_source, variables=element.__variables, first_pass=meta.first_pass)
 
             redundant_ref = source._load_ref()
 
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
index c39e09417..ffcae4993 100644
--- a/src/buildstream/plugins/sources/local.py
+++ b/src/buildstream/plugins/sources/local.py
@@ -49,11 +49,7 @@ class LocalSource(Source):
     BST_STAGE_VIRTUAL_DIRECTORY = True
     BST_KEY_REQUIRES_STAGE = True
 
-    def __init__(self, context, project, meta):
-        super().__init__(context, project, meta)
-
-        # Cached unique key to avoid multiple file system traversal if the unique key is requested multiple times.
-        self.__unique_key = None
+    __unique_key = None
 
     def configure(self, node):
         node.validate_keys(["path", *Source.COMMON_CONFIG_KEYS])
diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py
index 796f2b3d9..f1d965fa0 100644
--- a/src/buildstream/plugins/sources/workspace.py
+++ b/src/buildstream/plugins/sources/workspace.py
@@ -51,15 +51,12 @@ class WorkspaceSource(Source):
     BST_STAGE_VIRTUAL_DIRECTORY = True
     BST_KEY_REQUIRES_STAGE = True
 
-    def __init__(self, context, project, meta) -> None:
-        super().__init__(context, project, meta)
-
-        # Cached unique key
-        self.__unique_key = None
-        # the digest of the Directory following the import of the workspace
-        self.__digest = None
-        # the cache key of the last workspace build
-        self.__last_build = None
+    # Cached unique key
+    __unique_key = None
+    # the digest of the Directory following the import of the workspace
+    __digest = None
+    # the cache key of the last workspace build
+    __last_build = None
 
     def track(self) -> SourceRef:  # pylint: disable=arguments-differ
         return None
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index 7b790cdbb..049db7062 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -173,6 +173,7 @@ from ._cachekey import generate_key
 from .storage import CasBasedDirectory
 from .storage import FileBasedDirectory
 from .storage.directory import Directory, VirtualDirectoryError
+from ._variables import Variables
 
 if TYPE_CHECKING:
     from typing import Any, Dict, Set
@@ -321,6 +322,7 @@ class Source(Plugin):
         context: "Context",
         project: "Project",
         meta: MetaSource,
+        variables: Variables,
         *,
         alias_override: Optional[Tuple[str, str]] = None,
         unique_id: Optional[int] = None
@@ -341,6 +343,7 @@ class Source(Plugin):
         self.__element_kind = meta.element_kind  # The kind of the element owning this source
         self.__directory = meta.directory  # Staging relative directory
         self.__meta_kind = meta.kind  # The kind of this source, required for unpickling
+        self.__variables = variables  # The variables used to resolve the source's config
 
         self.__key = None  # Cache key for source
 
@@ -354,6 +357,8 @@ class Source(Plugin):
         # ask the element to configure itself.
         self.__init_defaults(project, meta)
         self.__config = self.__extract_config(meta)
+        variables.expand(self.__config)
+
         self.__first_pass = meta.first_pass
 
         # cached values for commonly access values on the source
@@ -1238,7 +1243,9 @@ class Source(Plugin):
 
         meta.first_pass = self.__first_pass
 
-        clone = source_kind(context, project, meta, alias_override=(alias, uri), unique_id=self._unique_id)
+        clone = source_kind(
+            context, project, meta, self.__variables, alias_override=(alias, uri), unique_id=self._unique_id
+        )
 
         # Do the necessary post instantiation routines here
         #
diff --git a/tests/sources/variables.py b/tests/sources/variables.py
new file mode 100644
index 000000000..3e9a08fd8
--- /dev/null
+++ b/tests/sources/variables.py
@@ -0,0 +1,35 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+
+import pytest
+
+from buildstream.exceptions import ErrorDomain, LoadErrorReason
+from buildstream.testing.runcli import cli  # pylint: disable=unused-import
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "variables")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_variables_are_resolved(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+    # Build, checkout
+    result = cli.run(project=project, args=["build", "target.bst"])
+    result.assert_success()
+    result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
+    result.assert_success()
+
+    # Check that the checkout contains the expected file
+    assert os.path.exists(os.path.join(checkoutdir, "file.txt"))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_handles_unresolved_variables(cli, tmpdir, datafiles):
+    project = str(datafiles)
+
+    result = cli.run(project=project, args=["build", "unresolveable-target.bst"])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE)
diff --git a/tests/sources/variables/folder/file.txt b/tests/sources/variables/folder/file.txt
new file mode 100644
index 000000000..a496efee8
--- /dev/null
+++ b/tests/sources/variables/folder/file.txt
@@ -0,0 +1 @@
+This is a text file
diff --git a/tests/sources/variables/project.conf b/tests/sources/variables/project.conf
new file mode 100644
index 000000000..dc34380a4
--- /dev/null
+++ b/tests/sources/variables/project.conf
@@ -0,0 +1,3 @@
+# Basic project
+name: foo
+min-version: 2.0
diff --git a/tests/sources/variables/target.bst b/tests/sources/variables/target.bst
new file mode 100644
index 000000000..0b492fdf3
--- /dev/null
+++ b/tests/sources/variables/target.bst
@@ -0,0 +1,9 @@
+kind: import
+description: This is the pony
+
+sources:
+- kind: local
+  path: "%{my-folder}/file.txt"
+
+variables:
+  my-folder: folder
diff --git a/tests/sources/variables/unresolveable-target.bst b/tests/sources/variables/unresolveable-target.bst
new file mode 100644
index 000000000..b6ce77363
--- /dev/null
+++ b/tests/sources/variables/unresolveable-target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: This is the pony
+
+sources:
+- kind: local
+  path: "%{my-folder}/file.txt"