diff options
31 files changed, 555 insertions, 258 deletions
diff --git a/doc/source/format_declaring.rst b/doc/source/format_declaring.rst index 7514d43e1..d8414c2bd 100644 --- a/doc/source/format_declaring.rst +++ b/doc/source/format_declaring.rst @@ -60,6 +60,60 @@ details here in order to have a more complete initial example. Let's break down the above and give a brief explanation of what these attributes mean. +.. _format_element_names: + +Element names and paths +~~~~~~~~~~~~~~~~~~~~~~~ +An *element name* is the filename of an element relative to the project's +:ref:`element path <project_element_path>`. + +Element names are the identifiers used to refer to elements, they are used +to specify an element's :ref:`dependencies <format_dependencies>`, to select +elements to build on the :ref:`command line <commands>`, and they are arbitrarily +used in various element specific configuration surfaces, for example the +*target* configuration of the :mod:`link <elements.link>` element is also +an *element name*. + + +Addressing elements +''''''''''''''''''' +When addressing elements in a single project, it is sufficient to use +the *element name* as a dependency or configuration parameter. + +When muliple projects are connected through :mod:`junction <elements.junction>` +elements, there is a need to address elements which are not in the same +project but in a junctioned *subproject*. In the case that you need to +address elements across junction boundaries, one must use *element paths*. + +An *element path* is a path to the element indicating the junction +elements leading up to the project, separated by ``:`` symbols, e.g.: +``junction.bst:element.bst``. + +Elements can be address across multiple junction boundaries with multiple +``:`` separators, e.g.: ``junction.bst:junction.bst:element.bst``. + + +Element naming rules +'''''''''''''''''''' +When naming the elements, use the following rules: + +* The name of the file must have ``.bst`` extension. + +* All characters in the name must be printable 7-bit ASCII characters. + +* Following characters are reserved and must not be part of the name: + + - ``<`` (less than) + - ``>`` (greater than) + - ``:`` (colon) + - ``"`` (double quote) + - ``/`` (forward slash) + - ``\`` (backslash) + - ``|`` (vertical bar) + - ``?`` (question mark) + - ``*`` (asterisk) + + Kind ~~~~ @@ -92,8 +146,8 @@ Depends - element2.bst Relationships between elements are specified with the ``depends`` attribute. Elements -may depend on other elements by specifying the :ref:`element path <project_element_path>` -relative filename to the elements they depend on here. +may depend on other elements by specifying the :ref:`element names <format_element_names>` +they depend on here. See :ref:`format_dependencies` for more information on the dependency model. @@ -228,7 +282,6 @@ Environment variables can be set to literal values here, these environment variables will be effective in the :mod:`Sandbox <buildstream.sandbox>` where build instructions are run for this element. - Environment variables can also be declared and overridden in the :ref:`projectconf` @@ -364,8 +417,7 @@ Attributes: * ``filename`` - The :ref:`element path <project_element_path>` relative filename of the element to - depend on in the project. + The :ref:`element name <format_element_names>` to depend on. * ``type`` @@ -375,15 +427,15 @@ Attributes: * ``junction`` - This attribute can be used to depend on elements in other projects. + This attribute can be used to specify the junction portion of the :ref:`element name <format_element_names>` + separately from the project local element name. - If a junction is specified, then it must be an :ref:`element path <project_element_path>` - relative filename of the junction element in the project. + This should be the *element name* of the :mod:`junction <elements.junction>` element + in the local project, possibly followed by other junctions in subprojects leading + to the project in which the element you want to depend on resides. - In the case that a *junction* is specified, the ``filename`` attribute indicates an element - in the *junctioned project*. - - See :mod:`junction <elements.junction>`. + In the case that a *junction* is specified, the ``filename`` attribute indicates an + element in the *junctioned project*. * ``strict`` @@ -398,33 +450,26 @@ Attributes: Cross-junction dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -As mentioned above, cross-junction dependencies can be specified using the -``junction`` attribute. They can also be expressed as simple strings as a -convenience shorthand. You can refer to cross-junction elements using the -syntax ``{junction-name}:{element-name}``. +As explained in the :ref:`element name <format_element_names>` section +on element addressing, elements can be addressed across junction boundaries +using *element paths* such as ``junction.bst:element.bst``. An element +at any depth can be specified by specifying multiple junction elements. -For example, the following is logically same as the example above: +For example, one can specify a subproject element dependency with +the following syntax: .. code:: yaml build-depends: - - baseproject.bst:foo.bst + - baseproject.bst:element.bst -Similarly, you can also refer to cross-junction elements via the ``filename`` -attribute, like so: +And one can specify an element residing in a sub-subproject as a +dependency like so: .. code:: yaml depends: - - filename: baseproject.bst:foo.bst - type: build - -.. note:: - - BuildStream does not allow recursive lookups for junction elements. If a - filename contains more than one ``:`` (colon) character, an error will be - raised. See :ref:`nested junctions <core_junction_nested>` for more details - on nested junctions. + - baseproject.bst:middleproject.bst:element.bst .. _format_dependencies_types: @@ -552,27 +597,3 @@ read-only variables are also dynamically declared by BuildStream: build, support for this is conditional on the element type and the build system used (any element using 'make' can implement this). - - -Naming elements ---------------- -When naming the element files, use the following rules: - -* The name of the file must have ``.bst`` extension. - -* All characters in the name must be printable 7-bit ASCII characters. - -* Following characters are reserved and must not be part of the name: - - - ``<`` (less than) - - ``>`` (greater than) - - ``:`` (colon) - - ``"`` (double quote) - - ``/`` (forward slash) - - ``\`` (backslash) - - ``|`` (vertical bar) - - ``?`` (question mark) - - ``*`` (asterisk) - -BuildStream will attempt to raise warnings when any of these rules are violated -but that may not always be possible. diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py index 7f4863e52..0c77e5fa1 100644 --- a/src/buildstream/_includes.py +++ b/src/buildstream/_includes.py @@ -150,7 +150,7 @@ class Includes: def _include_file(self, include, loader): shortname = include if ":" in include: - junction, include = include.split(":", 1) + junction, include = include.rsplit(":", 1) current_loader = loader.get_loader(junction) current_loader.project.ensure_fully_loaded() else: diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx index 2f4e7a0f9..014f01746 100644 --- a/src/buildstream/_loader/loadelement.pyx +++ b/src/buildstream/_loader/loadelement.pyx @@ -107,7 +107,7 @@ cdef class LoadElement: # if loader.project.junction: # dependency is in subproject, qualify name - self.full_name = '{}:{}'.format(loader.project.junction.name, self.name) + self.full_name = '{}:{}'.format(loader.project.junction._get_full_name(), self.name) else: # dependency is in top-level project self.full_name = self.name diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index b73f5b862..17f0d906f 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -160,191 +160,27 @@ class Loader: # get_loader(): # - # Return loader for specified junction + # Obtains the appropriate loader for the specified junction # # Args: - # filename (str): Junction name - # - # Raises: LoadError + # name (str): Name of junction, may have multiple `:` in the name + # rewritable (bool): Whether the loaded files should be rewritable + # this is a bit more expensive due to deep copies + # ticker (callable): An optional function for tracking load progress + # provenance (ProvenanceInformation): The provenance # - # Returns: A Loader or None if specified junction does not exist + # Returns: + # (Loader): loader for sub-project # - def get_loader(self, filename, *, rewritable=False, ticker=None, level=0, provenance=None): - - provenance_str = "" - if provenance is not None: - provenance_str = "{}: ".format(provenance) - - # return previously determined result - if filename in self._loaders: - loader = self._loaders[filename] - - if loader is None: - # do not allow junctions with the same name in different - # subprojects - raise LoadError( - "{}Conflicting junction {} in subprojects, define junction in {}".format( - provenance_str, filename, self.project.name - ), - LoadErrorReason.CONFLICTING_JUNCTION, - ) - - return loader - - if self._parent: - # junctions in the parent take precedence over junctions defined - # in subprojects - loader = self._parent.get_loader( - filename, rewritable=rewritable, ticker=ticker, level=level + 1, provenance=provenance - ) - if loader: - self._loaders[filename] = loader - return loader - - try: - self._load_file(filename, rewritable, ticker, provenance=provenance) - except LoadError as e: - if e.reason != LoadErrorReason.MISSING_FILE: - # other load error - raise - - if level == 0: - # junction element not found in this or ancestor projects - raise - - # mark junction as not available to allow detection of - # conflicting junctions in subprojects - self._loaders[filename] = None - return None - - # At this point we've loaded the LoadElement - load_element = self._elements[filename] - - # If the loaded element is a link, then just follow it - # immediately and move on to the target. - # - if load_element.link_target: - - _, filename, loader = self._parse_name(load_element.link_target, rewritable, ticker) - - if loader != self: - level = level + 1 - - return loader.get_loader( - filename, - rewritable=rewritable, - ticker=ticker, - level=level, - provenance=load_element.link_target_provenance, - ) + def get_loader(self, name, *, rewritable=False, ticker=None, level=0, provenance=None): + junction_path = name.split(":") + loader = self - # meta junction element - # - # Note that junction elements are not allowed to have - # dependencies, so disabling progress reporting here should - # have no adverse effects - the junction element itself cannot - # be depended on, so it would be confusing for its load to - # show up in logs. - # - # Any task counting *inside* the junction will be handled by - # its loader. - meta_element = self.collect_element_no_deps(self._elements[filename]) - if meta_element.kind != "junction": - raise LoadError( - "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind), - LoadErrorReason.INVALID_DATA, + for junction_name in junction_path: + loader = loader._get_loader( + junction_name, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance ) - # We check that junctions have no dependencies a little - # early. This is cheating, since we don't technically know - # that junctions aren't allowed to have dependencies. - # - # However, this makes progress reporting more intuitive - # because we don't need to load dependencies of an element - # that shouldn't have any, and therefore don't need to - # duplicate the load count for elements that shouldn't be. - # - # We also fail slightly earlier (since we don't need to go - # through the entire loading process), which is nice UX. It - # would be nice if this could be done for *all* element types, - # but since we haven't loaded those yet that's impossible. - if load_element.dependencies: - raise LoadError("Dependencies are forbidden for 'junction' elements", LoadErrorReason.INVALID_JUNCTION) - - element = Element._new_from_meta(meta_element) - element._initialize_state() - - # Handle the case where a subproject has no ref - # - if not element._has_all_sources_resolved(): - detail = "Try tracking the junction element with `bst source track {}`".format(filename) - raise LoadError( - "{}Subproject has no ref for junction: {}".format(provenance_str, filename), - LoadErrorReason.SUBPROJECT_INCONSISTENT, - detail=detail, - ) - - # Handle the case where a subproject needs to be fetched - # - if not element._has_all_sources_in_source_cache(): - if ticker: - ticker(filename, "Fetching subproject") - self._fetch_subprojects([element]) - - sources = list(element.sources()) - if len(sources) == 1 and sources[0]._get_local_path(): - # Optimization for junctions with a single local source - basedir = sources[0]._get_local_path() - else: - # Stage sources - element._set_required() - - # Note: We use _KeyStrength.WEAK here because junctions - # cannot have dependencies, therefore the keys are - # equivalent. - # - # Since the element has not necessarily been given a - # strong cache key at this point (in a non-strict build - # that is set *after* we complete building/pulling, which - # we haven't yet for this element), - # element._get_cache_key() can fail if used with the - # default _KeyStrength.STRONG. - basedir = os.path.join( - self.project.directory, ".bst", "staged-junctions", filename, element._get_cache_key(_KeyStrength.WEAK) - ) - if not os.path.exists(basedir): - os.makedirs(basedir, exist_ok=True) - element._stage_sources_at(basedir) - - # Load the project - project_dir = os.path.join(basedir, element.path) - try: - from .._project import Project # pylint: disable=cyclic-import - - project = Project( - project_dir, - self._context, - junction=element, - parent_loader=self, - search_for_project=False, - fetch_subprojects=self._fetch_subprojects, - ) - except LoadError as e: - if e.reason == LoadErrorReason.MISSING_PROJECT_CONF: - message = ( - provenance_str + "Could not find the project.conf file in the project " - "referred to by junction element '{}'.".format(element.name) - ) - if element.path: - message += " Was expecting it at path '{}' in the junction's source.".format(element.path) - raise LoadError(message=message, reason=LoadErrorReason.INVALID_JUNCTION) from e - - # Otherwise, we don't know the reason, so just raise - raise - - loader = project.loader - self._loaders[filename] = loader - return loader # get_state_for_child_job_pickling(self) @@ -562,7 +398,9 @@ class Loader: # If this element is a link then we need to resolve it # and replace the dependency we've processed with this one if top_element.link_target is not None: - _, filename, loader = self._parse_name(top_element.link_target, rewritable, ticker) + _, filename, loader = self._parse_name( + top_element.link_target, rewritable, ticker, top_element.link_target_provenance + ) top_element = loader._load_file(filename, rewritable, ticker, top_element.link_target_provenance) dependencies = extract_depends_from_node(top_element.node) @@ -607,7 +445,9 @@ class Loader: # If this dependency is a link then we need to resolve it # and replace the dependency we've processed with this one if dep_element.link_target: - _, filename, loader = self._parse_name(dep_element.link_target, rewritable, ticker) + _, filename, loader = self._parse_name( + dep_element.link_target, rewritable, ticker, dep_element.link_target_provenance + ) dep_element = loader._load_file(filename, rewritable, ticker, dep_element.link_target_provenance) # All is well, push the dependency onto the LoadElement @@ -725,6 +565,197 @@ class Loader: return self._meta_elements[top_element.name] + # _get_loader(): + # + # Return loader for specified junction + # + # Args: + # filename (str): Junction name + # + # Raises: LoadError + # + # Returns: A Loader or None if specified junction does not exist + # + def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0, provenance=None): + + provenance_str = "" + if provenance is not None: + provenance_str = "{}: ".format(provenance) + + # return previously determined result + if filename in self._loaders: + loader = self._loaders[filename] + + if loader is None: + # do not allow junctions with the same name in different + # subprojects + raise LoadError( + "{}Conflicting junction {} in subprojects, define junction in {}".format( + provenance_str, filename, self.project.name + ), + LoadErrorReason.CONFLICTING_JUNCTION, + ) + + return loader + + if self._parent: + # junctions in the parent take precedence over junctions defined + # in subprojects + loader = self._parent._get_loader( + filename, rewritable=rewritable, ticker=ticker, level=level + 1, provenance=provenance + ) + if loader: + self._loaders[filename] = loader + return loader + + try: + self._load_file(filename, rewritable, ticker, provenance=provenance) + except LoadError as e: + if e.reason != LoadErrorReason.MISSING_FILE: + # other load error + raise + + if level == 0: + # junction element not found in this or ancestor projects + raise + + # mark junction as not available to allow detection of + # conflicting junctions in subprojects + self._loaders[filename] = None + return None + + # At this point we've loaded the LoadElement + load_element = self._elements[filename] + + # If the loaded element is a link, then just follow it + # immediately and move on to the target. + # + if load_element.link_target: + + _, filename, loader = self._parse_name( + load_element.link_target, rewritable, ticker, provenance=load_element.link_target_provenance + ) + + if loader != self: + level = level + 1 + + return loader._get_loader( + filename, + rewritable=rewritable, + ticker=ticker, + level=level, + provenance=load_element.link_target_provenance, + ) + + # meta junction element + # + # Note that junction elements are not allowed to have + # dependencies, so disabling progress reporting here should + # have no adverse effects - the junction element itself cannot + # be depended on, so it would be confusing for its load to + # show up in logs. + # + # Any task counting *inside* the junction will be handled by + # its loader. + meta_element = self.collect_element_no_deps(self._elements[filename]) + if meta_element.kind != "junction": + raise LoadError( + "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind), + LoadErrorReason.INVALID_DATA, + ) + + # We check that junctions have no dependencies a little + # early. This is cheating, since we don't technically know + # that junctions aren't allowed to have dependencies. + # + # However, this makes progress reporting more intuitive + # because we don't need to load dependencies of an element + # that shouldn't have any, and therefore don't need to + # duplicate the load count for elements that shouldn't be. + # + # We also fail slightly earlier (since we don't need to go + # through the entire loading process), which is nice UX. It + # would be nice if this could be done for *all* element types, + # but since we haven't loaded those yet that's impossible. + if load_element.dependencies: + raise LoadError("Dependencies are forbidden for 'junction' elements", LoadErrorReason.INVALID_JUNCTION) + + element = Element._new_from_meta(meta_element) + element._initialize_state() + + # Handle the case where a subproject has no ref + # + if not element._has_all_sources_resolved(): + detail = "Try tracking the junction element with `bst source track {}`".format(filename) + raise LoadError( + "{}Subproject has no ref for junction: {}".format(provenance_str, filename), + LoadErrorReason.SUBPROJECT_INCONSISTENT, + detail=detail, + ) + + # Handle the case where a subproject needs to be fetched + # + if not element._has_all_sources_in_source_cache(): + if ticker: + ticker(filename, "Fetching subproject") + self._fetch_subprojects([element]) + + sources = list(element.sources()) + if len(sources) == 1 and sources[0]._get_local_path(): + # Optimization for junctions with a single local source + basedir = sources[0]._get_local_path() + else: + # Stage sources + element._set_required() + + # Note: We use _KeyStrength.WEAK here because junctions + # cannot have dependencies, therefore the keys are + # equivalent. + # + # Since the element has not necessarily been given a + # strong cache key at this point (in a non-strict build + # that is set *after* we complete building/pulling, which + # we haven't yet for this element), + # element._get_cache_key() can fail if used with the + # default _KeyStrength.STRONG. + basedir = os.path.join( + self.project.directory, ".bst", "staged-junctions", filename, element._get_cache_key(_KeyStrength.WEAK) + ) + if not os.path.exists(basedir): + os.makedirs(basedir, exist_ok=True) + element._stage_sources_at(basedir) + + # Load the project + project_dir = os.path.join(basedir, element.path) + try: + from .._project import Project # pylint: disable=cyclic-import + + project = Project( + project_dir, + self._context, + junction=element, + parent_loader=self, + search_for_project=False, + fetch_subprojects=self._fetch_subprojects, + ) + except LoadError as e: + if e.reason == LoadErrorReason.MISSING_PROJECT_CONF: + message = ( + provenance_str + "Could not find the project.conf file in the project " + "referred to by junction element '{}'.".format(element.name) + ) + if element.path: + message += " Was expecting it at path '{}' in the junction's source.".format(element.path) + raise LoadError(message=message, reason=LoadErrorReason.INVALID_JUNCTION) from e + + # Otherwise, we don't know the reason, so just raise + raise + + loader = project.loader + self._loaders[filename] = loader + + return loader + # _parse_name(): # # Get junction and base name of element along with loader for the sub-project @@ -734,13 +765,14 @@ class Loader: # rewritable (bool): Whether the loaded files should be rewritable # this is a bit more expensive due to deep copies # ticker (callable): An optional function for tracking load progress + # provenance (ProvenanceInformation): The provenance # # Returns: # (tuple): - (str): name of the junction element # - (str): name of the element # - (Loader): loader for sub-project # - def _parse_name(self, name, rewritable, ticker): + def _parse_name(self, name, rewritable, ticker, provenance=None): # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. @@ -748,8 +780,7 @@ class Loader: if len(junction_path) == 1: return None, junction_path[-1], self else: - self._load_file(junction_path[-2], rewritable, ticker) - loader = self.get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker) + loader = self.get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker, provenance=provenance) return junction_path[-2], junction_path[-1], loader # Print a warning message, checks warning_token against project configuration diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx index 1f264789a..70f262a10 100644 --- a/src/buildstream/_loader/types.pyx +++ b/src/buildstream/_loader/types.pyx @@ -133,15 +133,9 @@ cdef class Dependency: "junction attribute is specified.".format(self.provenance, self.name), LoadErrorReason.INVALID_DATA) - # Name of the element should never contain more than one `:` characters - if self.name.count(':') > 1: - raise LoadError("{}: Dependency {} contains multiple `:` in its name. " - "Recursive lookups for cross-junction elements is not " - "allowed.".format(self.provenance, self.name), LoadErrorReason.INVALID_DATA) - # Attempt to split name if no junction was specified explicitly - if not self.junction and self.name.count(':') == 1: - self.junction, self.name = self.name.split(':') + if not self.junction and ':' in self.name: + self.junction, self.name = self.name.rsplit(':', maxsplit=1) # _extract_depends_from_node(): diff --git a/src/buildstream/_pluginfactory/pluginoriginjunction.py b/src/buildstream/_pluginfactory/pluginoriginjunction.py index 7c887e4cb..8c1d560fb 100644 --- a/src/buildstream/_pluginfactory/pluginoriginjunction.py +++ b/src/buildstream/_pluginfactory/pluginoriginjunction.py @@ -35,7 +35,7 @@ class PluginOriginJunction(PluginOrigin): # Get access to the project indicated by the junction, # possibly loading it as a side effect. # - loader = self.project.loader.get_loader(self._junction) + loader = self.project.loader.get_loader(self._junction, provenance=self.provenance) project = loader.project project.ensure_fully_loaded() diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py index 6795043e7..f8652e5cb 100644 --- a/src/buildstream/plugin.py +++ b/src/buildstream/plugin.py @@ -755,7 +755,7 @@ class Plugin: # Set the name, depending on element or source plugin type name = self._element_name if self.__type_tag == "source" else self.name # pylint: disable=no-member if project.junction: - return "{}:{}".format(project.junction.name, name) + return "{}:{}".format(project.junction._get_full_name(), name) else: return name diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index c9e78632f..3e221cce7 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -63,15 +63,15 @@ Overview links to other projects and are not in the dependency graph on their own. With a junction element in place, local elements can depend on elements in -the other BuildStream project using the additional ``junction`` attribute in the -dependency dictionary: +the other BuildStream project using :ref:`element paths <format_element_names>`. +For example, if you have a ``toolchain.bst`` junction element referring to +a project which contains a ``gcc.bst`` element, you can express a build +dependency to the compiler like this: .. code:: yaml - depends: - - junction: toolchain.bst - filename: gcc.bst - type: build + build-depends: + - junction: toolchain.bst:gcc.bst While junctions are elements, only a limited set of element operations is supported. They can be tracked and fetched like other elements. diff --git a/src/buildstream/plugins/elements/link.py b/src/buildstream/plugins/elements/link.py index 611108241..e6d7f056e 100644 --- a/src/buildstream/plugins/elements/link.py +++ b/src/buildstream/plugins/elements/link.py @@ -27,7 +27,7 @@ a symbolic element which will be resolved to another element. Overview -------- The only configuration allowed in a ``link`` element is the specified -target of the link. +target :ref:`element name <format_element_names>` of the link. .. code:: yaml @@ -37,7 +37,8 @@ target of the link. target: element.bst The ``link`` element can be used to refer to elements in subprojects, and -can be used to symbolically link junctions as well as other elements. +can be used to symbolically link :mod:`junction <elements.junction>` elements +as well as other elements. """ from buildstream import Element diff --git a/tests/format/include.py b/tests/format/include.py index 12b043c8e..5c273e1a0 100644 --- a/tests/format/include.py +++ b/tests/format/include.py @@ -310,3 +310,24 @@ def test_option_from_deep_junction(cli, tmpdir, datafiles): result.assert_success() loaded = _yaml.load_data(result.output) assert not loaded.get_bool("is-default") + + +@pytest.mark.datafiles(DATA_DIR) +def test_include_full_path(cli, tmpdir, datafiles): + project = os.path.join(str(datafiles), "full_path") + + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) + result.assert_success() + loaded = _yaml.load_data(result.output) + assert loaded.get_str("bar") == "red" + assert loaded.get_str("foo") == "blue" + + +@pytest.mark.datafiles(DATA_DIR) +def test_include_invalid_full_path(cli, tmpdir, datafiles): + project = os.path.join(str(datafiles), "full_path") + + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "invalid.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + # Make sure the root cause provenance is in the output. + assert "invalid.bst [line 4 column 7]" in result.stderr diff --git a/tests/format/include/full_path/elements/element.bst b/tests/format/include/full_path/elements/element.bst new file mode 100644 index 000000000..7f6f08b32 --- /dev/null +++ b/tests/format/include/full_path/elements/element.bst @@ -0,0 +1,6 @@ +kind: manual + +variables: + (@): + - subproject.bst:sub.yaml + - subproject.bst:subsubproject-junction.bst:subsub.yaml diff --git a/tests/format/include/full_path/elements/invalid.bst b/tests/format/include/full_path/elements/invalid.bst new file mode 100644 index 000000000..0b5240db0 --- /dev/null +++ b/tests/format/include/full_path/elements/invalid.bst @@ -0,0 +1,4 @@ +kind: manual + +variables: + (@): subproject.bst:subsubproject-junction.bst:pony.yaml diff --git a/tests/format/include/full_path/elements/subproject.bst b/tests/format/include/full_path/elements/subproject.bst new file mode 100644 index 000000000..6664eeec6 --- /dev/null +++ b/tests/format/include/full_path/elements/subproject.bst @@ -0,0 +1,5 @@ +kind: junction + +sources: +- kind: local + path: subproject diff --git a/tests/format/include/full_path/project.conf b/tests/format/include/full_path/project.conf new file mode 100644 index 000000000..4e2fb0063 --- /dev/null +++ b/tests/format/include/full_path/project.conf @@ -0,0 +1,4 @@ +name: simple +min-version: 2.0 + +element-path: elements diff --git a/tests/format/include/full_path/subproject/elements/subsubproject-junction.bst b/tests/format/include/full_path/subproject/elements/subsubproject-junction.bst new file mode 100644 index 000000000..018fb8ec4 --- /dev/null +++ b/tests/format/include/full_path/subproject/elements/subsubproject-junction.bst @@ -0,0 +1,5 @@ +kind: junction + +sources: +- kind: local + path: subsubproject diff --git a/tests/format/include/full_path/subproject/files/hello.txt b/tests/format/include/full_path/subproject/files/hello.txt new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/tests/format/include/full_path/subproject/files/hello.txt @@ -0,0 +1 @@ +hello diff --git a/tests/format/include/full_path/subproject/project.conf b/tests/format/include/full_path/subproject/project.conf new file mode 100644 index 000000000..1529ece04 --- /dev/null +++ b/tests/format/include/full_path/subproject/project.conf @@ -0,0 +1,4 @@ +name: subproject +min-version: 2.0 + +element-path: elements diff --git a/tests/format/include/full_path/subproject/sub.yaml b/tests/format/include/full_path/subproject/sub.yaml new file mode 100644 index 000000000..db3359e57 --- /dev/null +++ b/tests/format/include/full_path/subproject/sub.yaml @@ -0,0 +1 @@ +bar: "red" diff --git a/tests/format/include/full_path/subproject/subsubproject/project.conf b/tests/format/include/full_path/subproject/subsubproject/project.conf new file mode 100644 index 000000000..3b470ccf2 --- /dev/null +++ b/tests/format/include/full_path/subproject/subsubproject/project.conf @@ -0,0 +1,2 @@ +name: subsubproject +min-version: 2.0 diff --git a/tests/format/include/full_path/subproject/subsubproject/subsub.yaml b/tests/format/include/full_path/subproject/subsubproject/subsub.yaml new file mode 100644 index 000000000..4c78a1a57 --- /dev/null +++ b/tests/format/include/full_path/subproject/subsubproject/subsub.yaml @@ -0,0 +1 @@ +foo: "blue" diff --git a/tests/format/junctions.py b/tests/format/junctions.py index f097e0b8b..b60d16816 100644 --- a/tests/format/junctions.py +++ b/tests/format/junctions.py @@ -417,3 +417,55 @@ def test_junction_show(cli, tmpdir, datafiles): # Show, assert that it says junction assert cli.get_element_state(project, "base.bst") == "junction" + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("target", ["junction-full-path.bst", "element-full-path.bst", "foo.bst:base.bst:target.bst"]) +def test_full_path(cli, tmpdir, datafiles, target): + project_foo = os.path.join(str(datafiles), "foo") + copy_subprojects(project_foo, datafiles, ["base"]) + + project = os.path.join(str(datafiles), "toplevel") + copy_subprojects(project, datafiles, ["base", "foo", "bar"]) + + checkoutdir = os.path.join(str(tmpdir), "checkout") + + # FIXME: This file can be removed after removing the junction coalescing feature + os.remove(os.path.join(project, "base.bst")) + + # Build, checkout + result = cli.run(project=project, args=["build", target]) + result.assert_success() + result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) + result.assert_success() + + # Check that the checkout contains the expected file from base + assert os.path.exists(os.path.join(checkoutdir, "base.txt")) + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize( + "target,provenance", + [ + ("junction-full-path-notfound.bst", "junction-full-path-notfound.bst [line 3 column 2]"), + ("element-full-path-notfound.bst", "element-full-path-notfound.bst [line 3 column 2]"), + ("foo.bst:base.bst:pony.bst", None), + ], +) +def test_full_path_not_found(cli, tmpdir, datafiles, target, provenance): + project_foo = os.path.join(str(datafiles), "foo") + copy_subprojects(project_foo, datafiles, ["base"]) + + project = os.path.join(str(datafiles), "toplevel") + copy_subprojects(project, datafiles, ["base", "foo", "bar"]) + + # FIXME: This file can be removed after removing the junction coalescing feature + os.remove(os.path.join(project, "base.bst")) + + # Build + result = cli.run(project=project, args=["build", target]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + + # Check that provenance was provided if expected + if provenance: + assert provenance in result.stderr diff --git a/tests/format/junctions/toplevel/element-full-path-notfound.bst b/tests/format/junctions/toplevel/element-full-path-notfound.bst new file mode 100644 index 000000000..55efaca10 --- /dev/null +++ b/tests/format/junctions/toplevel/element-full-path-notfound.bst @@ -0,0 +1,3 @@ +kind: stack +depends: +- foo.bst:base.bst:pony.bst diff --git a/tests/format/junctions/toplevel/element-full-path.bst b/tests/format/junctions/toplevel/element-full-path.bst new file mode 100644 index 000000000..f58559a76 --- /dev/null +++ b/tests/format/junctions/toplevel/element-full-path.bst @@ -0,0 +1,3 @@ +kind: stack +depends: +- foo.bst:base.bst:target.bst diff --git a/tests/format/junctions/toplevel/junction-full-path-notfound.bst b/tests/format/junctions/toplevel/junction-full-path-notfound.bst new file mode 100644 index 000000000..a57d6ba76 --- /dev/null +++ b/tests/format/junctions/toplevel/junction-full-path-notfound.bst @@ -0,0 +1,4 @@ +kind: stack +depends: +- junction: foo.bst:base.bst + filename: pony.bst diff --git a/tests/format/junctions/toplevel/junction-full-path.bst b/tests/format/junctions/toplevel/junction-full-path.bst new file mode 100644 index 000000000..4a4f67d19 --- /dev/null +++ b/tests/format/junctions/toplevel/junction-full-path.bst @@ -0,0 +1,4 @@ +kind: stack +depends: +- junction: foo.bst:base.bst + filename: target.bst diff --git a/tests/format/link.py b/tests/format/link.py index 47e19b90c..e2c9e0b84 100644 --- a/tests/format/link.py +++ b/tests/format/link.py @@ -59,7 +59,9 @@ def test_conditional_link(cli, tmpdir, datafiles, target, greeting, expected_fil # Test links to junctions from local projects and subprojects # @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target", ["target-local.bst", "target-nested.bst"]) +@pytest.mark.parametrize( + "target", ["target-local.bst", "target-nested.bst", "full-path-link.bst", "target-full-path.bst"] +) def test_simple_junctions(cli, tmpdir, datafiles, target): project = os.path.join(str(datafiles), "simple-junctions") checkoutdir = os.path.join(str(tmpdir), "checkout") @@ -115,6 +117,10 @@ def test_conditional_junctions(cli, tmpdir, datafiles, greeting, expected_file): ("linked-local-junction.bst", "subproject-link-notfound.bst [line 4 column 10]",), # Depends on an element via a link to a non-existing subproject junction ("linked-nested-junction.bst", "subsubproject-link-notfound.bst [line 4 column 10]",), + # Target is a link to a non-existing nested element referred to with a full path + ("link-full-path.bst", "link-full-path.bst [line 4 column 10]"), + # Target depends on a link to a non-existing nested element referred to with a full path + ("target-full-path.bst", "link-full-path.bst [line 4 column 10]"), ], ) def test_link_not_found(cli, tmpdir, datafiles, target, provenance): diff --git a/tests/format/link/notfound/elements/link-full-path.bst b/tests/format/link/notfound/elements/link-full-path.bst new file mode 100644 index 000000000..a619c549a --- /dev/null +++ b/tests/format/link/notfound/elements/link-full-path.bst @@ -0,0 +1,4 @@ +kind: link + +config: + target: subproject.bst:subsubproject-junction.bst:pony.bst diff --git a/tests/format/link/notfound/elements/target-full-path.bst b/tests/format/link/notfound/elements/target-full-path.bst new file mode 100644 index 000000000..ad67c343e --- /dev/null +++ b/tests/format/link/notfound/elements/target-full-path.bst @@ -0,0 +1,4 @@ +kind: stack + +depends: +- link-full-path.bst diff --git a/tests/format/link/simple-junctions/elements/full-path-link.bst b/tests/format/link/simple-junctions/elements/full-path-link.bst new file mode 100644 index 000000000..306e18cb2 --- /dev/null +++ b/tests/format/link/simple-junctions/elements/full-path-link.bst @@ -0,0 +1,4 @@ +kind: link + +config: + target: subproject.bst:subsubproject-junction.bst:hello.bst diff --git a/tests/format/link/simple-junctions/elements/target-full-path.bst b/tests/format/link/simple-junctions/elements/target-full-path.bst new file mode 100644 index 000000000..3fdc9519a --- /dev/null +++ b/tests/format/link/simple-junctions/elements/target-full-path.bst @@ -0,0 +1,4 @@ +kind: stack + +depends: +- full-path-link.bst diff --git a/tests/plugins/loading.py b/tests/plugins/loading.py index 63a2ca3d4..7aeb242f3 100644 --- a/tests/plugins/loading.py +++ b/tests/plugins/loading.py @@ -10,7 +10,7 @@ import os import shutil import pytest -from buildstream.exceptions import ErrorDomain +from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream.testing import cli # pylint: disable=unused-import from buildstream import _yaml @@ -599,3 +599,111 @@ def test_junction_pip_plugin_version_conflict(cli, datafiles, plugin_type): result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "junction-plugin-load-error") + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) +def test_junction_full_path_found(cli, datafiles, plugin_type): + project = str(datafiles) + subproject = os.path.join(project, "subproject") + subsubproject = os.path.join(subproject, "subsubproject") + + shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) + + update_project( + project, + { + "plugins": [ + { + "origin": "junction", + "junction": "subproject-junction.bst:subsubproject-junction.bst", + plugin_type: ["found"], + } + ] + }, + ) + update_project( + subsubproject, + { + "plugins": [ + {"origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"],} + ] + }, + ) + setup_element(project, plugin_type, "found") + + result = cli.run(project=project, args=["show", "element.bst"]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) +def test_junction_full_path_not_found(cli, datafiles, plugin_type): + project = str(datafiles) + subproject = os.path.join(project, "subproject") + subsubproject = os.path.join(subproject, "subsubproject") + + shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) + + # The toplevel says to search for the "notfound" plugin in the subproject + # + update_project( + project, + { + "plugins": [ + { + "origin": "junction", + "junction": "subproject-junction.bst:subsubproject-junction.bst", + plugin_type: ["notfound"], + } + ] + }, + ) + + # The subsubproject only configures the "found" plugin + # + update_project( + subsubproject, + { + "plugins": [ + {"origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"],} + ] + }, + ) + setup_element(project, plugin_type, "notfound") + + result = cli.run(project=project, args=["show", "element.bst"]) + result.assert_main_error(ErrorDomain.PLUGIN, "junction-plugin-not-found") + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize( + "plugin_type,provenance", + [("elements", "project.conf [line 10 column 2]"), ("sources", "project.conf [line 10 column 2]")], +) +def test_junction_invalid_full_path(cli, datafiles, plugin_type, provenance): + project = str(datafiles) + subproject = os.path.join(project, "subproject") + subsubproject = os.path.join(subproject, "subsubproject") + + shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) + + # The toplevel says to search for the "notfound" plugin in the subproject + # + update_project( + project, + { + "plugins": [ + { + "origin": "junction", + "junction": "subproject-junction.bst:pony-junction.bst", + plugin_type: ["notfound"], + } + ] + }, + ) + setup_element(project, plugin_type, "notfound") + + result = cli.run(project=project, args=["show", "element.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) + assert provenance in result.stderr |