diff options
24 files changed, 115 insertions, 121 deletions
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py index 64ee8fe57..56f6d68dc 100644 --- a/src/buildstream/_basecache.py +++ b/src/buildstream/_basecache.py @@ -79,7 +79,7 @@ class BaseCache(): try: artifacts = config_node.get_sequence(cls.config_node_name, default=[]) except LoadError: - provenance = _yaml.node_get_provenance(config_node, key=cls.config_node_name) + provenance = config_node.get_node(cls.config_node_name).get_provenance() raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, "%s: 'artifacts' must be a single 'url:' mapping, or a list of mappings" % (str(provenance))) diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py index dac92aa2a..f0c84f7b6 100644 --- a/src/buildstream/_cas/casremote.py +++ b/src/buildstream/_cas/casremote.py @@ -8,7 +8,6 @@ import uuid import grpc -from .. import _yaml from .._protos.google.rpc import code_pb2 from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc @@ -35,7 +34,7 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key url = spec_node.get_str('url') push = spec_node.get_bool('push', default=False) if not url: - provenance = _yaml.node_get_provenance(spec_node, 'url') + provenance = spec_node.get_node('url').get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: empty artifact cache URL".format(provenance)) @@ -54,12 +53,12 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert = os.path.join(basedir, client_cert) if client_key and not client_cert: - provenance = _yaml.node_get_provenance(spec_node, 'client-key') + provenance = spec_node.get_node('client-key').get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: 'client-key' was specified without 'client-cert'".format(provenance)) if client_cert and not client_key: - provenance = _yaml.node_get_provenance(spec_node, 'client-cert') + provenance = spec_node.get_node('client-cert').get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: 'client-cert' was specified without 'client-key'".format(provenance)) diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 9cb4f5943..2243192b3 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -507,9 +507,10 @@ class Context(): # LoadError, when the value is not of the expected type, or is not found. # def _node_get_option_str(node, key, allowed_options): - result = node.get_str(key) + result_node = node.get_scalar(key) + result = result_node.as_str() if result not in allowed_options: - provenance = _yaml.node_get_provenance(node, key) + provenance = result_node.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: {} should be one of: {}".format( provenance, key, ", ".join(allowed_options))) diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py index 4c1eabb7e..678f24dba 100644 --- a/src/buildstream/_includes.py +++ b/src/buildstream/_includes.py @@ -43,7 +43,6 @@ class Includes: else: includes = includes_node.as_str_list() - include_provenance = _yaml.node_get_provenance(node, key='(@)') del node['(@)'] for include in reversed(includes): @@ -53,6 +52,7 @@ class Includes: include_node, file_path, sub_loader = self._include_file(include, current_loader) except LoadError as e: + include_provenance = includes_node.get_provenance() if e.reason == LoadErrorReason.MISSING_FILE: message = "{}: Include block references a file that could not be found: '{}'.".format( include_provenance, include) @@ -65,6 +65,7 @@ class Includes: raise if file_path in included: + include_provenance = includes_node.get_provenance() raise LoadError(LoadErrorReason.RECURSIVE_INCLUDE, "{}: trying to recursively include {}". format(include_provenance, file_path)) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index e45381fbf..27975dc34 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -463,7 +463,7 @@ class Loader(): return meta_element node = element.node - elt_provenance = _yaml.node_get_provenance(node) + elt_provenance = node.get_provenance() meta_sources = [] sources = node.get_sequence(Symbol.SOURCES, default=[]) diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx index cd206cfb4..fe1cea789 100644 --- a/src/buildstream/_loader/types.pyx +++ b/src/buildstream/_loader/types.pyx @@ -69,7 +69,7 @@ cdef class Dependency: str default_dep_type=None): cdef str dep_type - self.provenance = _yaml.node_get_provenance(dep) + self.provenance = dep.get_provenance() if type(dep) is _yaml.ScalarNode: self.name = dep.as_str() @@ -88,7 +88,7 @@ cdef class Dependency: if dep_type is None or dep_type == <str> Symbol.ALL: dep_type = None elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]: - provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE) + provenance = dep.get_scalar(Symbol.TYPE).get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'" .format(provenance, dep_type)) diff --git a/src/buildstream/_options/optionarch.py b/src/buildstream/_options/optionarch.py index 75be1a8ff..612ca2aa0 100644 --- a/src/buildstream/_options/optionarch.py +++ b/src/buildstream/_options/optionarch.py @@ -17,7 +17,6 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -from .. import _yaml from .._exceptions import LoadError, LoadErrorReason, PlatformError from .._platform import Platform from .optionenum import OptionEnum @@ -55,7 +54,7 @@ class OptionArch(OptionEnum): # Do not terminate the loop early to ensure we validate # all values in the list. except PlatformError as e: - provenance = _yaml.node_get_provenance(node.get_sequence('values').scalar_at(index)) + provenance = node.get_sequence('values').scalar_at(index).get_provenance() prefix = "" if provenance: prefix = "{}: ".format(provenance) diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py index 477fe1ed9..d1a7a85c9 100644 --- a/src/buildstream/_options/optionenum.py +++ b/src/buildstream/_options/optionenum.py @@ -17,7 +17,6 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -from .. import _yaml from .._exceptions import LoadError, LoadErrorReason from .option import Option, OPTION_SYMBOLS @@ -50,16 +49,17 @@ class OptionEnum(Option): if not self.values: raise LoadError(LoadErrorReason.INVALID_DATA, "{}: No values specified for {} option '{}'" - .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name)) + .format(node.get_provenance(), self.OPTION_TYPE, self.name)) # Allow subclass to define the default value self.value = self.load_default_value(node) def load_value(self, node, *, transform=None): - self.value = node.get_str(self.name) + value_node = node.get_scalar(self.name) + self.value = value_node.as_str() if transform: self.value = transform(self.value) - self.validate(self.value, _yaml.node_get_provenance(node, self.name)) + self.validate(self.value, value_node) def set_value(self, value): self.validate(value) @@ -68,17 +68,20 @@ class OptionEnum(Option): def get_value(self): return self.value - def validate(self, value, provenance=None): + def validate(self, value, node=None): if value not in self.values: - prefix = "" - if provenance: + if node is not None: + provenance = node.get_provenance() prefix = "{}: ".format(provenance) + else: + prefix = "" raise LoadError(LoadErrorReason.INVALID_DATA, "{}Invalid value for {} option '{}': {}\n" .format(prefix, self.OPTION_TYPE, self.name, value) + "Valid values: {}".format(", ".join(self.values))) def load_default_value(self, node): - value = node.get_str('default') - self.validate(value, _yaml.node_get_provenance(node, 'default')) + value_node = node.get_scalar('default') + value = value_node.as_str() + self.validate(value, value_node) return value diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py index 1d86361d8..80dd1b55d 100644 --- a/src/buildstream/_options/optionflags.py +++ b/src/buildstream/_options/optionflags.py @@ -17,7 +17,6 @@ # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -from .. import _yaml from .._exceptions import LoadError, LoadErrorReason from .option import Option, OPTION_SYMBOLS @@ -51,17 +50,19 @@ class OptionFlags(Option): if not self.values: raise LoadError(LoadErrorReason.INVALID_DATA, "{}: No values specified for {} option '{}'" - .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name)) + .format(node.get_provenance(), self.OPTION_TYPE, self.name)) - self.value = node.get_sequence('default', default=[]).as_str_list() - self.validate(self.value, _yaml.node_get_provenance(node, 'default')) + value_node = node.get_sequence('default', default=[]) + self.value = value_node.as_str_list() + self.validate(self.value, value_node) def load_value(self, node, *, transform=None): - self.value = node.get_sequence(self.name).as_str_list() + value_node = node.get_sequence(self.name) + self.value = value_node.as_str_list() if transform: self.value = [transform(x) for x in self.value] self.value = sorted(self.value) - self.validate(self.value, _yaml.node_get_provenance(node, self.name)) + self.validate(self.value, value_node) def set_value(self, value): # Strip out all whitespace, allowing: "value1, value2 , value3" @@ -76,12 +77,14 @@ class OptionFlags(Option): def get_value(self): return ",".join(self.value) - def validate(self, value, provenance=None): + def validate(self, value, node=None): for flag in value: if flag not in self.values: - prefix = "" - if provenance: + if node is not None: + provenance = node.get_provenance() prefix = "{}: ".format(provenance) + else: + prefix = "" raise LoadError(LoadErrorReason.INVALID_DATA, "{}Invalid value for flags option '{}': {}\n" .format(prefix, self.name, value) + diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py index cd4bfb6dc..3b33c0349 100644 --- a/src/buildstream/_options/optionpool.py +++ b/src/buildstream/_options/optionpool.py @@ -74,7 +74,7 @@ class OptionPool(): try: opt_type = _OPTION_TYPES[opt_type_name] except KeyError: - p = _yaml.node_get_provenance(option_definition, 'type') + p = option_definition.get_scalar('type').get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Invalid option type '{}'".format(p, opt_type_name)) @@ -90,11 +90,11 @@ class OptionPool(): # node (dict): The loaded YAML options # def load_yaml_values(self, node, *, transform=None): - for option_name in node.keys(): + for option_name, option_value in node.items(): try: option = self._options[option_name] except KeyError as e: - p = _yaml.node_get_provenance(node, option_name) + p = option_value.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Unknown option '{}' specified" .format(p, option_name)) from e @@ -256,7 +256,7 @@ class OptionPool(): # assertion in a given dictionary, and not lose an assertion due to # it being overwritten by a later assertion which might also trigger. if assertion is not None: - p = _yaml.node_get_provenance(node, '(!)') + p = node.get_scalar('(!)').get_provenance() raise LoadError(LoadErrorReason.USER_ASSERTION, "{}: {}".format(p, assertion.strip())) @@ -266,7 +266,7 @@ class OptionPool(): for condition in conditions: tuples = list(condition.items()) if len(tuples) > 1: - provenance = _yaml.node_get_provenance(condition) + provenance = condition.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Conditional statement has more than one key".format(provenance)) @@ -275,11 +275,11 @@ class OptionPool(): apply_fragment = self._evaluate(expression) except LoadError as e: # Prepend the provenance of the error - provenance = _yaml.node_get_provenance(condition) + provenance = condition.get_provenance() raise LoadError(e.reason, "{}: {}".format(provenance, e)) from e if type(value) is not _yaml.MappingNode: # pylint: disable=unidiomatic-typecheck - provenance = _yaml.node_get_provenance(condition) + provenance = condition.get_provenance() raise LoadError(LoadErrorReason.ILLEGAL_COMPOSITE, "{}: Only values of type 'dict' can be composed.".format(provenance)) diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 45eae03f9..f075aeb51 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -268,14 +268,14 @@ class Project(): full_path = self._absolute_directory_path / path if full_path.is_symlink(): - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND, "{}: Specified path '{}' must not point to " "symbolic links " .format(provenance, path_str)) if path.parts and path.parts[0] == '..': - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID, "{}: Specified path '{}' first component must " "not be '..'" @@ -287,7 +287,7 @@ class Project(): else: full_resolved_path = full_path.resolve(strict=True) # pylint: disable=unexpected-keyword-arg except FileNotFoundError: - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.MISSING_FILE, "{}: Specified path '{}' does not exist" .format(provenance, path_str)) @@ -296,14 +296,14 @@ class Project(): full_resolved_path == self._absolute_directory_path) if not is_inside: - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID, "{}: Specified path '{}' must not lead outside of the " "project directory" .format(provenance, path_str)) if path.is_absolute(): - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID, "{}: Absolute path: '{}' invalid.\n" "Please specify a path relative to the project's root." @@ -312,20 +312,20 @@ class Project(): if full_resolved_path.is_socket() or ( full_resolved_path.is_fifo() or full_resolved_path.is_block_device()): - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND, "{}: Specified path '{}' points to an unsupported " "file kind" .format(provenance, path_str)) if check_is_file and not full_resolved_path.is_file(): - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND, "{}: Specified path '{}' is not a regular file" .format(provenance, path_str)) if check_is_dir and not full_resolved_path.is_dir(): - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() raise LoadError(LoadErrorReason.PROJ_PATH_INVALID_KIND, "{}: Specified path '{}' is not a directory" .format(provenance, path_str)) @@ -627,9 +627,10 @@ class Project(): ignore_unknown=True) # Use separate file for storing source references - self.ref_storage = pre_config_node.get_str('ref-storage') + ref_storage_node = pre_config_node.get_scalar('ref-storage') + self.ref_storage = ref_storage_node.as_str() if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]: - p = _yaml.node_get_provenance(pre_config_node, 'ref-storage') + p = ref_storage_node.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Invalid value '{}' specified for ref-storage" .format(p, self.ref_storage)) diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py index cee74dc11..10dd000b2 100644 --- a/src/buildstream/_projectrefs.py +++ b/src/buildstream/_projectrefs.py @@ -63,7 +63,7 @@ class ProjectRefs(): def load(self, options): try: self._toplevel_node = _yaml.load(self._fullpath, shortname=self._base_name, copy_tree=True) - provenance = _yaml.node_get_provenance(self._toplevel_node) + provenance = self._toplevel_node.get_provenance() self._toplevel_save = provenance.toplevel # Process any project options immediately diff --git a/src/buildstream/_variables.pyx b/src/buildstream/_variables.pyx index ba8338f23..eb2deb553 100644 --- a/src/buildstream/_variables.pyx +++ b/src/buildstream/_variables.pyx @@ -139,7 +139,7 @@ cdef class Variables: for var in expstr[1::2]: if var not in self._expstr_map: line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}" - provenance = _yaml.node_get_provenance(self.original, key) + provenance = expstr.get_provenance() summary.append(line.format(unmatched=var, variable=key, provenance=provenance)) if summary: raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, @@ -153,7 +153,7 @@ cdef class Variables: continue if var in visited: raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE, - "{}: ".format(_yaml.node_get_provenance(self.original, var)) + + "{}: ".format(self.original.get_scalar(var).get_provenance()) + ("Variable '{}' expands to contain a reference to itself. " + "Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var)) visited.append(var) diff --git a/src/buildstream/_yaml.pxd b/src/buildstream/_yaml.pxd index 7b5209077..9c67f1ad2 100644 --- a/src/buildstream/_yaml.pxd +++ b/src/buildstream/_yaml.pxd @@ -27,6 +27,7 @@ cdef class Node: cdef int column cpdef Node copy(self) + cpdef ProvenanceInformation get_provenance(self) cpdef object strip_node_info(self) cpdef void _assert_fully_composited(self) except * @@ -89,6 +90,3 @@ cdef class ProvenanceInformation: cdef public int col, line cdef public object project, toplevel cdef public bint is_synthetic - - -cpdef ProvenanceInformation node_get_provenance(Node node, str key=*) diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx index a84c05759..945a82378 100644 --- a/src/buildstream/_yaml.pyx +++ b/src/buildstream/_yaml.pyx @@ -94,6 +94,9 @@ cdef class Node: cpdef Node copy(self): raise NotImplementedError() + cpdef ProvenanceInformation get_provenance(self): + return ProvenanceInformation(self) + cpdef object strip_node_info(self): raise NotImplementedError() @@ -171,7 +174,7 @@ cdef class ScalarNode(Node): elif self.value in ('False', 'false'): return False else: - provenance = node_get_provenance(self) + provenance = self.get_provenance() path = provenance.toplevel._find(self)[-1] raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type '{}'" @@ -181,7 +184,7 @@ cdef class ScalarNode(Node): try: return int(self.value) except ValueError: - provenance = node_get_provenance(self) + provenance = self.get_provenance() path = provenance.toplevel._find(self)[-1] raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type '{}'" @@ -205,8 +208,8 @@ cdef class ScalarNode(Node): if target_value is not None and type(target_value) is not ScalarNode: raise CompositeError(path, "{}: Cannot compose scalar on non-scalar at {}".format( - node_get_provenance(self), - node_get_provenance(target_value))) + self.get_provenance(), + target_value.get_provenance())) target.value[key] = self @@ -263,7 +266,7 @@ cdef class MappingNode(Node): try: self._composite(target, []) except CompositeError as e: - source_provenance = node_get_provenance(self) + source_provenance = self.get_provenance() error_prefix = "" if source_provenance: error_prefix = "{}: ".format(source_provenance) @@ -292,7 +295,7 @@ cdef class MappingNode(Node): if value is _sentinel: if default is _sentinel: - provenance = node_get_provenance(self) + provenance = self.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Dictionary did not contain expected key '{}'".format(provenance, key)) @@ -308,7 +311,7 @@ cdef class MappingNode(Node): value = self.get(key, default, MappingNode) if type(value) is not MappingNode and value is not None: - provenance = node_get_provenance(value) + provenance = value.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type 'Mapping'" .format(provenance, key)) @@ -322,12 +325,12 @@ cdef class MappingNode(Node): if allow_none: return None - provenance = node_get_provenance(self) + provenance = self.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Dictionary did not contain expected key '{}'".format(provenance, key)) if allowed_types and type(value) not in allowed_types: - provenance = node_get_provenance(self) + provenance = self.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not one of the following: {}.".format( provenance, key, ", ".join(allowed_types))) @@ -341,7 +344,7 @@ cdef class MappingNode(Node): if value is None: value = ScalarNode.__new__(ScalarNode, self.file_index, 0, next_synthetic_counter(), None) else: - provenance = node_get_provenance(value) + provenance = value.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type 'Scalar'" .format(provenance, key)) @@ -352,7 +355,7 @@ cdef class MappingNode(Node): value = self.get(key, default, SequenceNode) if type(value) is not SequenceNode and value is not None: - provenance = node_get_provenance(value) + provenance = value.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type 'Sequence'" .format(provenance, key)) @@ -403,7 +406,7 @@ cdef class MappingNode(Node): for key in self.value: if key not in valid_keys_set: - provenance = node_get_provenance(self, key=key) + provenance = self.get_node(key).get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Unexpected key: {}".format(provenance, key)) @@ -445,8 +448,8 @@ cdef class MappingNode(Node): # Else composing on top of normal dict or a scalar, so raise... raise CompositeError(path, "{}: Cannot compose lists onto {}".format( - node_get_provenance(self), - node_get_provenance(target_value))) + self.get_provenance(), + target_value.get_provenance())) else: # We're composing a dict into target now if key not in target.value: @@ -514,7 +517,7 @@ cdef class MappingNode(Node): has_keys = True if has_keys and has_directives: - provenance = node_get_provenance(self) + provenance = self.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Dictionary contains array composition directives and arbitrary keys" .format(provenance)) @@ -566,7 +569,7 @@ cdef class MappingNode(Node): # never existed in the underlying data # if key in ('(>)', '(<)', '(=)'): - provenance = node_get_provenance(value) + provenance = value.get_provenance() raise LoadError(LoadErrorReason.TRAILING_LIST_DIRECTIVE, "{}: Attempt to override non-existing list".format(provenance)) @@ -613,7 +616,7 @@ cdef class SequenceNode(Node): value = self.value[index] if type(value) is not MappingNode: - provenance = node_get_provenance(self) + provenance = self.get_provenance() path = ["[{}]".format(p) for p in provenance.toplevel._find(self)] + ["[{}]".format(index)] raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type '{}'" @@ -624,7 +627,7 @@ cdef class SequenceNode(Node): cdef value = self.value[index] if allowed_types and type(value) not in allowed_types: - provenance = node_get_provenance(self) + provenance = self.get_provenance() raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not one of the following: {}.".format( provenance, index, ", ".join(allowed_types))) @@ -635,7 +638,7 @@ cdef class SequenceNode(Node): value = self.value[index] if type(value) is not ScalarNode: - provenance = node_get_provenance(self) + provenance = self.get_provenance() path = ["[{}]".format(p) for p in provenance.toplevel._find(self)] + ["[{}]".format(index)] raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type '{}'" @@ -646,7 +649,7 @@ cdef class SequenceNode(Node): value = self.value[index] if type(value) is not SequenceNode: - provenance = node_get_provenance(self) + provenance = self.get_provenance() path = ["[{}]".format(p) for p in provenance.toplevel._find(self)] + ["[{}]".format(index)] raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Value of '{}' is not of the expected type '{}'" @@ -675,9 +678,9 @@ cdef class SequenceNode(Node): target_value._is_composite_list()): raise CompositeError(path, "{}: List cannot overwrite {} at: {}" - .format(node_get_provenance(self), + .format(self.get_provenance(), key, - node_get_provenance(target_value))) + target_value.get_provenance())) # Looks good, clobber it target.value[key] = self @@ -763,7 +766,7 @@ cdef int next_synthetic_counter(): return __counter -# Returned from node_get_provenance +# Returned from Node.get_provenance cdef class ProvenanceInformation: def __init__(self, Node nodeish): @@ -1126,24 +1129,6 @@ cpdef Node load_data(str data, int file_index=_SYNTHETIC_FILE_INDEX, str file_na return contents -# node_get_provenance() -# -# Gets the provenance for a node -# -# Args: -# node (Node): a dictionary -# key (str): key in the dictionary -# -# Returns: The Provenance of the dict, member or list element -# -cpdef ProvenanceInformation node_get_provenance(Node node, str key=None): - if key is None: - # Retrieving the provenance for this node directly - return ProvenanceInformation(node) - - return ProvenanceInformation((<MappingNode> node).value.get(key)) - - # new_synthetic_file() # # Create a new synthetic mapping node, with an associated file entry @@ -1251,7 +1236,7 @@ def assert_symbol_name(str symbol_name, str purpose, *, Node ref_node=None, bint message = "Invalid symbol name for {}: '{}'".format(purpose, symbol_name) if ref_node: - provenance = node_get_provenance(ref_node) + provenance = ref_node.get_provenance() if provenance is not None: message = "{}: {}".format(provenance, message) diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 35d25562d..446b20352 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -514,7 +514,7 @@ class Element(Plugin): try: return self.__variables.subst(value) except LoadError as e: - provenance = _yaml.node_get_provenance(node, key=member_name) + provenance = node.get_scalar(member_name).get_provenance() raise LoadError(e.reason, '{}: {}'.format(provenance, e), detail=e.detail) from e def node_subst_list(self, node, member_name): @@ -536,7 +536,7 @@ class Element(Plugin): try: ret.append(self.__variables.subst(value.as_str())) except LoadError as e: - provenance = _yaml.node_get_provenance(value) + provenance = value.get_provenance() raise LoadError(e.reason, '{}: {}'.format(provenance, e), detail=e.detail) from e return ret @@ -2600,8 +2600,13 @@ class Element(Plugin): variables._assert_fully_composited() for var in ('project-name', 'element-name', 'max-jobs'): - provenance = _yaml.node_get_provenance(variables, var) - if provenance and not provenance.is_synthetic: + node = variables.get_node(var, allow_none=True) + + if node is None: + continue + + provenance = node.get_provenance() + if not provenance.is_synthetic: raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED, "{}: invalid redefinition of protected variable '{}'" .format(provenance, var)) diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py index 2e5cc9678..92e52504d 100644 --- a/src/buildstream/plugin.py +++ b/src/buildstream/plugin.py @@ -116,7 +116,6 @@ import sys from contextlib import contextmanager from weakref import WeakValueDictionary -from . import _yaml from . import utils from ._exceptions import PluginError, ImplError from ._message import Message, MessageType @@ -360,7 +359,7 @@ class Plugin(): Returns: (str): A string describing the provenance of the node and member """ - provenance = _yaml.node_get_provenance(node, key=member_name) + provenance = node.get_node(member_name).get_provenance() return str(provenance) def node_get_project_path(self, node, *, diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index ca3c5c3d9..76b1a6270 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -114,7 +114,7 @@ class SandboxRemote(Sandbox): def require_node(config, keyname): val = config.get_mapping(keyname, default=None) if val is None: - provenance = _yaml.node_get_provenance(remote_config, key=keyname) + provenance = remote_config.get_provenance() raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, "{}: '{}' was not present in the remote " "execution configuration (remote-execution). " @@ -146,7 +146,7 @@ class SandboxRemote(Sandbox): if 'execution-service' not in remote_config: exec_config = _yaml.Node.from_dict({'url': remote_config['url']}) else: - provenance = _yaml.node_get_provenance(remote_config, key='url') + provenance = remote_config.get_node('url').get_provenance() raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, "{}: 'url' and 'execution-service' keys were found in the remote " "execution configuration (remote-execution). " @@ -164,7 +164,7 @@ class SandboxRemote(Sandbox): for config_key, config in zip(service_keys, service_configs): # Either both or none of the TLS client key/cert pair must be specified: if ('client-key' in config) != ('client-cert' in config): - provenance = _yaml.node_get_provenance(remote_config, key=config_key) + provenance = remote_config.get_node(config_key).get_provenance() raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, "{}: TLS client key/cert pair is incomplete. " "You must specify both 'client-key' and 'client-cert' " diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 855045040..5125169e8 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -307,7 +307,7 @@ class Source(Plugin): """ def __init__(self, context, project, meta, *, alias_override=None, unique_id=None): - provenance = _yaml.node_get_provenance(meta.config) + provenance = meta.config.get_provenance() super().__init__("{}-{}".format(meta.element_name, meta.element_index), context, project, provenance, "source", unique_id=unique_id) @@ -955,9 +955,9 @@ class Source(Plugin): for key, action in actions.items(): # Obtain the top level node and its file if action == 'add': - provenance = _yaml.node_get_provenance(node) + provenance = node.get_provenance() else: - provenance = _yaml.node_get_provenance(node, key=key) + provenance = node.get_node(key).get_provenance() toplevel_node = provenance.toplevel diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 07ebe7e6a..d3eec0d21 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -462,7 +462,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname='junction-dep.bst') ref_node = element_node.get_sequence('depends').mapping_at(0) - provenance = _yaml.node_get_provenance(ref_node) + provenance = ref_node.get_provenance() assert str(provenance) in result.stderr diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py index ba79dcfd1..7ea357ac2 100644 --- a/tests/frontend/fetch.py +++ b/tests/frontend/fetch.py @@ -172,5 +172,5 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname='junction-dep.bst') ref_node = element_node.get_sequence('depends').mapping_at(0) - provenance = _yaml.node_get_provenance(ref_node) + provenance = ref_node.get_provenance() assert str(provenance) in result.stderr diff --git a/tests/frontend/show.py b/tests/frontend/show.py index a0421bd37..756fe1786 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -335,7 +335,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname='junction-dep.bst') ref_node = element_node.get_sequence('depends').mapping_at(0) - provenance = _yaml.node_get_provenance(ref_node) + provenance = ref_node.get_provenance() assert str(provenance) in dep_result.stderr dep_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) diff --git a/tests/frontend/track.py b/tests/frontend/track.py index b8ab0d4eb..a628043d8 100644 --- a/tests/frontend/track.py +++ b/tests/frontend/track.py @@ -277,7 +277,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname='junction-dep.bst') ref_node = element_node.get_sequence('depends').mapping_at(0) - provenance = _yaml.node_get_provenance(ref_node) + provenance = ref_node.get_provenance() assert str(provenance) in result.stderr @@ -316,7 +316,7 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage): # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname='junction-dep.bst') ref_node = element_node.get_sequence('depends').mapping_at(0) - provenance = _yaml.node_get_provenance(ref_node) + provenance = ref_node.get_provenance() assert str(provenance) in result.stderr # Now track the junction itself diff --git a/tests/internals/yaml.py b/tests/internals/yaml.py index 8ceed9d15..9e62cdcb3 100644 --- a/tests/internals/yaml.py +++ b/tests/internals/yaml.py @@ -24,8 +24,8 @@ def test_load_yaml(datafiles): assert loaded.get_str('kind') == 'pony' -def assert_provenance(filename, line, col, node, key=None): - provenance = _yaml.node_get_provenance(node, key=key) +def assert_provenance(filename, line, col, node): + provenance = node.get_provenance() assert isinstance(provenance, _yaml.ProvenanceInformation) @@ -56,7 +56,7 @@ def test_member_provenance(datafiles): loaded = _yaml.load(filename) assert loaded.get_str('kind') == 'pony' - assert_provenance(filename, 2, 13, loaded, 'description') + assert_provenance(filename, 2, 13, loaded.get_scalar('description')) @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -108,7 +108,7 @@ def test_node_get(datafiles): assert len(children) == 7 child = base.get_sequence('children').mapping_at(6) - assert_provenance(filename, 20, 8, child, 'mood') + assert_provenance(filename, 20, 8, child.get_scalar('mood')) extra = base.get_mapping('extra') with pytest.raises(LoadError) as exc: @@ -256,7 +256,7 @@ def test_list_composition(datafiles, filename, tmpdir, child = children.mapping_at(index) assert child.get_str('mood') == mood - assert_provenance(prov_file, prov_line, prov_col, child, 'mood') + assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood')) # Test that overwriting a list with an empty list works as expected. @@ -378,7 +378,7 @@ def test_list_composition_twice(datafiles, tmpdir, filename1, filename2, child = children.mapping_at(index) assert child.get_str('mood') == mood - assert_provenance(prov_file, prov_line, prov_col, child, 'mood') + assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood')) ##################### # Round 2 - Fight ! @@ -395,7 +395,7 @@ def test_list_composition_twice(datafiles, tmpdir, filename1, filename2, child = children.mapping_at(index) assert child.get_str('mood') == mood - assert_provenance(prov_file, prov_line, prov_col, child, 'mood') + assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood')) @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -494,7 +494,7 @@ def test_node_find_target(datafiles, case): # are not the same nodes as in `prov.toplevel` loaded = _yaml.load(filename, copy_tree=True) - prov = _yaml.node_get_provenance(loaded) + prov = loaded.get_provenance() toplevel = prov.toplevel |