diff options
| author | Benjamin Schubert <contact@benschubert.me> | 2019-06-26 14:11:52 +0000 |
|---|---|---|
| committer | Benjamin Schubert <contact@benschubert.me> | 2019-06-26 14:11:52 +0000 |
| commit | 024b1ddfadbb71c6f10caa553a10885827b9366f (patch) | |
| tree | e4fed04f093eba0d09964f2e38e6249292fb0eed | |
| parent | 950d7e90a2d124254093a0f05a9b80ab6a8ed95c (diff) | |
| parent | 9d84e70482a94dfdefbe2a1bfb14fe43459d6324 (diff) | |
| download | buildstream-024b1ddfadbb71c6f10caa553a10885827b9366f.tar.gz | |
Merge branch 'bschubert/node-api-noget' into 'bschubert/new-node-api'
introduce a Node api, and remove node_get
See merge request BuildStream/buildstream!1401
70 files changed, 703 insertions, 757 deletions
diff --git a/doc/bst2html.py b/doc/bst2html.py index 448ab1389..c6dcf3673 100755 --- a/doc/bst2html.py +++ b/doc/bst2html.py @@ -332,9 +332,9 @@ def run_session(description, tempdir, source_cache, palette, config_file, force) # if not force: needs_build = False - commands = _yaml.node_get(desc, list, 'commands') + commands = desc.get_sequence('commands') for command in commands: - output = _yaml.node_get(command, str, 'output', default_value=None) + output = command.get_str('output', default=None) if output is not None and check_needs_build(desc_dir, output, force=False): needs_build = True break @@ -347,7 +347,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force) # tarball. This workaround lets us build docs from # a source distribution tarball. # - symlinks = _yaml.node_get(desc, dict, 'workaround-symlinks', default_value={}) + symlinks = desc.get_mapping('workaround-symlinks', default={}) for symlink, target in _yaml.node_items(symlinks): # Resolve real path to where symlink should be @@ -367,7 +367,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force) # not a source distribution, no need to complain pass - remove_files = _yaml.node_get(desc, list, 'remove-files', default_value=[]) + remove_files = desc.get_sequence('remove-files', default=[]).as_str_list() for remove_file in remove_files: remove_file = os.path.join(desc_dir, remove_file) remove_file = os.path.realpath(remove_file) @@ -379,23 +379,21 @@ def run_session(description, tempdir, source_cache, palette, config_file, force) # Run commands # - commands = _yaml.node_get(desc, list, 'commands') - for c in commands: - command = _yaml.node_get(desc, dict, 'commands', indices=[commands.index(c)]) - + commands = desc.get_sequence('commands') + for command in commands: # Get the directory where this command should be run - directory = _yaml.node_get(command, str, 'directory') + directory = command.get_str('directory') directory = os.path.join(desc_dir, directory) directory = os.path.realpath(directory) # Get the command string - command_str = _yaml.node_get(command, str, 'command') + command_str = command.get_str('command') # Check whether this is a shell command and not a bst command - is_shell = _yaml.node_get(command, bool, 'shell', default_value=False) + is_shell = command.get_bool('shell', default=False) # Check if there is fake output - command_fake_output = _yaml.node_get(command, str, 'fake-output', default_value=None) + command_fake_output = command.get_str('fake-output', default=None) # Run the command, or just use the fake output if command_fake_output is None: @@ -407,7 +405,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force) command_out = command_fake_output # Encode and save the output if that was asked for - output = _yaml.node_get(command, str, 'output', default_value=None) + output = command.get_str('output', default=None) if output is not None: # Convert / Generate a nice <div> converted = generate_html(command_out, directory, config_file, diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py index ef9414787..e8923c7b1 100644 --- a/src/buildstream/_basecache.py +++ b/src/buildstream/_basecache.py @@ -74,10 +74,10 @@ class BaseCache(): cache_specs = [] try: - artifacts = [_yaml.node_get(config_node, dict, cls.config_node_name)] + artifacts = [config_node.get_mapping(cls.config_node_name)] except LoadError: try: - artifacts = _yaml.node_get(config_node, list, cls.config_node_name, default_value=[]) + artifacts = config_node.get_sequence(cls.config_node_name, default=[]) except LoadError: provenance = _yaml.node_get_provenance(config_node, key=cls.config_node_name) raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py index cd46e9c38..56cfea20f 100644 --- a/src/buildstream/_cas/casremote.py +++ b/src/buildstream/_cas/casremote.py @@ -32,24 +32,24 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key @staticmethod def _new_from_config_node(spec_node, basedir=None): _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance-name']) - url = _yaml.node_get(spec_node, str, 'url') - push = _yaml.node_get(spec_node, bool, 'push', default_value=False) + url = spec_node.get_str('url') + push = spec_node.get_bool('push', default=False) if not url: provenance = _yaml.node_get_provenance(spec_node, 'url') raise LoadError(LoadErrorReason.INVALID_DATA, "{}: empty artifact cache URL".format(provenance)) - instance_name = _yaml.node_get(spec_node, str, 'instance-name', default_value=None) + instance_name = spec_node.get_str('instance-name', default=None) - server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None) + server_cert = spec_node.get_str('server-cert', default=None) if server_cert and basedir: server_cert = os.path.join(basedir, server_cert) - client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None) + client_key = spec_node.get_str('client-key', default=None) if client_key and basedir: client_key = os.path.join(basedir, client_key) - client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None) + client_cert = spec_node.get_str('client-cert', default=None) if client_cert and basedir: client_cert = os.path.join(basedir, client_cert) diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 3f6e6ac3c..21c272daa 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -222,7 +222,7 @@ class Context(): # Allow the ~ tilde expansion and any environment variables in # path specification in the config files. # - path = _yaml.node_get(defaults, str, directory) + path = defaults.get_str(directory) path = os.path.expanduser(path) path = os.path.expandvars(path) path = os.path.normpath(path) @@ -251,10 +251,10 @@ class Context(): # Load quota configuration # We need to find the first existing directory in the path of our # cachedir - the cachedir may not have been created yet. - cache = _yaml.node_get(defaults, dict, 'cache') + cache = defaults.get_mapping('cache') _yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees']) - self.config_cache_quota_string = _yaml.node_get(cache, str, 'quota') + self.config_cache_quota_string = cache.get_str('quota') try: self.config_cache_quota = utils._parse_size(self.config_cache_quota_string, self.casdir) @@ -273,42 +273,43 @@ class Context(): self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults) # Load pull build trees configuration - self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees') + self.pull_buildtrees = cache.get_bool('pull-buildtrees') # Load cache build trees configuration self.cache_buildtrees = _node_get_option_str( cache, 'cache-buildtrees', ['always', 'auto', 'never']) # Load logging config - logging = _yaml.node_get(defaults, dict, 'logging') + logging = defaults.get_mapping('logging') _yaml.node_validate(logging, [ 'key-length', 'verbose', 'error-lines', 'message-lines', 'debug', 'element-format', 'message-format' ]) - self.log_key_length = _yaml.node_get(logging, int, 'key-length') - self.log_debug = _yaml.node_get(logging, bool, 'debug') - self.log_verbose = _yaml.node_get(logging, bool, 'verbose') - self.log_error_lines = _yaml.node_get(logging, int, 'error-lines') - self.log_message_lines = _yaml.node_get(logging, int, 'message-lines') - self.log_element_format = _yaml.node_get(logging, str, 'element-format') - self.log_message_format = _yaml.node_get(logging, str, 'message-format') + self.log_key_length = logging.get_int('key-length') + self.log_debug = logging.get_bool('debug') + self.log_verbose = logging.get_bool('verbose') + self.log_error_lines = logging.get_int('error-lines') + self.log_message_lines = logging.get_int('message-lines') + self.log_message_lines = logging.get_int('message-lines') + self.log_element_format = logging.get_str('element-format') + self.log_message_format = logging.get_str('message-format') # Load scheduler config - scheduler = _yaml.node_get(defaults, dict, 'scheduler') + scheduler = defaults.get_mapping('scheduler') _yaml.node_validate(scheduler, [ 'on-error', 'fetchers', 'builders', 'pushers', 'network-retries' ]) self.sched_error_action = _node_get_option_str( scheduler, 'on-error', ['continue', 'quit', 'terminate']) - self.sched_fetchers = _yaml.node_get(scheduler, int, 'fetchers') - self.sched_builders = _yaml.node_get(scheduler, int, 'builders') - self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers') - self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries') + self.sched_fetchers = scheduler.get_int('fetchers') + self.sched_builders = scheduler.get_int('builders') + self.sched_pushers = scheduler.get_int('pushers') + self.sched_network_retries = scheduler.get_int('network-retries') # Load per-projects overrides - self._project_overrides = _yaml.node_get(defaults, dict, 'projects', default_value={}) + self._project_overrides = defaults.get_mapping('projects', default={}) # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. @@ -398,17 +399,16 @@ class Context(): # get_overrides(): # # Fetch the override dictionary for the active project. This returns - # a node loaded from YAML and as such, values loaded from the returned - # node should be loaded using the _yaml.node_get() family of functions. + # a node loaded from YAML. # # Args: # project_name (str): The project name # # Returns: - # (dict): The overrides dictionary for the specified project + # (MappingNode): The overrides dictionary for the specified project # def get_overrides(self, project_name): - return _yaml.node_get(self._project_overrides, dict, project_name, default_value={}) + return self._project_overrides.get_mapping(project_name, default={}) # get_strict(): # @@ -423,7 +423,7 @@ class Context(): # so work out if we should be strict, and then cache the result toplevel = self.get_toplevel_project() overrides = self.get_overrides(toplevel.name) - self._strict_build_plan = _yaml.node_get(overrides, bool, 'strict', default_value=True) + self._strict_build_plan = overrides.get_bool('strict', default=True) # If it was set by the CLI, it overrides any config # Ditto if we've already computed this, then we return the computed @@ -740,7 +740,7 @@ class Context(): # _node_get_option_str() # -# Like _yaml.node_get(), but also checks value is one of the allowed option +# Like Node.get_scalar().as_str(), but also checks value is one of the allowed option # strings. Fetches a value from a dictionary node, and makes sure it's one of # the pre-defined options. # @@ -756,7 +756,7 @@ class Context(): # LoadError, when the value is not of the expected type, or is not found. # def _node_get_option_str(node, key, allowed_options): - result = _yaml.node_get(node, str, key) + result = node.get_str(key) if result not in allowed_options: provenance = _yaml.node_get_provenance(node, key) raise LoadError(LoadErrorReason.INVALID_DATA, diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 53adb188c..f2f149403 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -100,7 +100,7 @@ def complete_target(args, incomplete): return [] # The project is not required to have an element-path - element_directory = _yaml.node_get(project, str, 'element-path', default_value='') + element_directory = project.get_str('element-path', default='') # If a project was loaded, use its element-path to # adjust our completion's base directory diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py index 1f539d820..4837877f4 100644 --- a/src/buildstream/_gitsourcebase.py +++ b/src/buildstream/_gitsourcebase.py @@ -376,25 +376,25 @@ class _GitSourceBase(Source): BST_MIRROR_CLASS = _GitMirror def configure(self, node): - ref = self.node_get_member(node, str, 'ref', None) + ref = node.get_str('ref', None) config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format', 'track-tags', 'tags'] self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS) - tags_node = self.node_get_member(node, list, 'tags', []) + tags_node = node.get_sequence('tags', []) for tag_node in tags_node: self.node_validate(tag_node, ['tag', 'commit', 'annotated']) tags = self._load_tags(node) - self.track_tags = self.node_get_member(node, bool, 'track-tags', False) + self.track_tags = node.get_bool('track-tags', default=False) - self.original_url = self.node_get_member(node, str, 'url') + self.original_url = node.get_str('url') self.mirror = self.BST_MIRROR_CLASS(self, '', self.original_url, ref, tags=tags, primary=True) - self.tracking = self.node_get_member(node, str, 'track', None) + self.tracking = node.get_str('track', None) - self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1') + self.ref_format = node.get_str('ref-format', 'sha1') if self.ref_format not in ['sha1', 'git-describe']: provenance = self.node_provenance(node, member_name='ref-format') raise SourceError("{}: Unexpected value for ref-format: {}".format(provenance, self.ref_format)) @@ -405,17 +405,17 @@ class _GitSourceBase(Source): raise SourceError("{}: Git sources require a ref and/or track".format(self), reason="missing-track-and-ref") - self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True) + self.checkout_submodules = node.get_bool('checkout-submodules', default=True) self.submodules = [] # Parse a dict of submodule overrides, stored in the submodule_overrides # and submodule_checkout_overrides dictionaries. self.submodule_overrides = {} self.submodule_checkout_overrides = {} - modules = self.node_get_member(node, dict, 'submodules', {}) + modules = node.get_mapping('submodules', {}) for path, _ in self.node_items(modules): - submodule = self.node_get_member(modules, dict, path) - url = self.node_get_member(submodule, str, 'url', None) + submodule = modules.get_mapping(path) + url = submodule.get_str('url', None) # Make sure to mark all URLs that are specified in the configuration if url: @@ -423,7 +423,7 @@ class _GitSourceBase(Source): self.submodule_overrides[path] = url if 'checkout' in submodule: - checkout = self.node_get_member(submodule, bool, 'checkout') + checkout = submodule.get_bool('checkout') self.submodule_checkout_overrides[path] = checkout self.mark_download_url(self.original_url) @@ -464,7 +464,7 @@ class _GitSourceBase(Source): return Consistency.INCONSISTENT def load_ref(self, node): - self.mirror.ref = self.node_get_member(node, str, 'ref', None) + self.mirror.ref = node.get_str('ref', None) self.mirror.tags = self._load_tags(node) def get_ref(self): @@ -663,11 +663,11 @@ class _GitSourceBase(Source): def _load_tags(self, node): tags = [] - tags_node = self.node_get_member(node, list, 'tags', []) + tags_node = node.get_sequence('tags', []) for tag_node in tags_node: - tag = self.node_get_member(tag_node, str, 'tag') - commit_ref = self.node_get_member(tag_node, str, 'commit') - annotated = self.node_get_member(tag_node, bool, 'annotated') + tag = tag_node.get_str('tag') + commit_ref = tag_node.get_str('commit') + annotated = tag_node.get_bool('annotated') tags.append((tag, commit_ref, annotated)) return tags diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py index 8f507b566..2831c9765 100644 --- a/src/buildstream/_includes.py +++ b/src/buildstream/_includes.py @@ -35,17 +35,14 @@ class Includes: if current_loader is None: current_loader = self._loader - includes = _yaml.node_get(node, None, '(@)', default_value=None) - if isinstance(includes, str): - includes = [includes] + includes_node = node.get_node('(@)', allowed_types=[_yaml.ScalarNode, _yaml.SequenceNode], allow_none=True) - if not isinstance(includes, list) and includes is not None: - provenance = _yaml.node_get_provenance(node, key='(@)') - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: {} must either be list or str".format(provenance, includes)) + if includes_node: + if type(includes_node) is _yaml.ScalarNode: # pylint: disable=unidiomatic-typecheck + includes = [includes_node.as_str()] + else: + includes = includes_node.as_str_list() - include_provenance = None - if includes: include_provenance = _yaml.node_get_provenance(node, key='(@)') _yaml.node_del(node, '(@)') diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index fa3539b22..1bad5fe0b 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -227,7 +227,7 @@ class Loader(): message, detail=detail) from e else: raise - kind = _yaml.node_get(node, str, Symbol.KIND) + kind = node.get_str(Symbol.KIND) if kind == "junction": self._first_pass_options.process_node(node) else: @@ -306,7 +306,7 @@ class Loader(): dep_deps = extract_depends_from_node(dep_element.node) loader_queue.append((dep_element, list(reversed(dep_deps)), [])) - if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction': + if dep_element.node.get_str(Symbol.KIND) == 'junction': raise LoadError(LoadErrorReason.INVALID_DATA, "{}: Cannot depend on junction" .format(dep.provenance)) @@ -466,33 +466,29 @@ class Loader(): elt_provenance = _yaml.node_get_provenance(node) meta_sources = [] - sources = _yaml.node_get(node, list, Symbol.SOURCES, default_value=[]) - element_kind = _yaml.node_get(node, str, Symbol.KIND) + sources = node.get_sequence(Symbol.SOURCES, default=[]) + element_kind = node.get_str(Symbol.KIND) - # Safe loop calling into _yaml.node_get() for each element ensures - # we have good error reporting - for i in range(len(sources)): - source = _yaml.node_get(node, dict, Symbol.SOURCES, indices=[i]) - kind = _yaml.node_get(source, str, Symbol.KIND) + for index, source in enumerate(sources): + kind = source.get_str(Symbol.KIND) _yaml.node_del(source, Symbol.KIND) # Directory is optional - directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None) + directory = source.get_str(Symbol.DIRECTORY, default=None) if directory: _yaml.node_del(source, Symbol.DIRECTORY) - index = sources.index(source) meta_source = MetaSource(element.name, index, element_kind, kind, source, directory) meta_sources.append(meta_source) meta_element = MetaElement(self.project, element.name, element_kind, elt_provenance, meta_sources, - _yaml.node_get(node, dict, Symbol.CONFIG, default_value={}), - _yaml.node_get(node, dict, Symbol.VARIABLES, default_value={}), - _yaml.node_get(node, dict, Symbol.ENVIRONMENT, default_value={}), - _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]), - _yaml.node_get(node, dict, Symbol.PUBLIC, default_value={}), - _yaml.node_get(node, dict, Symbol.SANDBOX, default_value={}), + node.get_mapping(Symbol.CONFIG, default={}), + node.get_mapping(Symbol.VARIABLES, default={}), + node.get_mapping(Symbol.ENVIRONMENT, default={}), + node.get_sequence(Symbol.ENV_NOCACHE, default=[]).as_str_list(), + node.get_mapping(Symbol.PUBLIC, default={}), + node.get_mapping(Symbol.SANDBOX, default={}), element_kind == 'junction') # Cache it now, make sure it's already there before recursing diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx index da33d6c54..dfd8f9046 100644 --- a/src/buildstream/_loader/types.pyx +++ b/src/buildstream/_loader/types.pyx @@ -72,12 +72,12 @@ cdef class Dependency: self.provenance = provenance - if type(dep) is str: - self.name = <str> dep + if type(dep) is _yaml.ScalarNode: + self.name = dep.as_str() self.dep_type = default_dep_type self.junction = None - elif type(dep) is _yaml.Node and type(dep.value) is dict: + elif type(dep) is _yaml.MappingNode: if default_dep_type: _yaml.node_validate(<_yaml.Node> dep, ['filename', 'junction']) dep_type = default_dep_type @@ -85,7 +85,7 @@ cdef class Dependency: _yaml.node_validate(<_yaml.Node> dep, ['filename', 'type', 'junction']) # Make type optional, for this we set it to None - dep_type = <str> _yaml.node_get(<_yaml.Node> dep, str, <str> Symbol.TYPE, None, None) + dep_type = (<_yaml.MappingNode> dep).get_str(<str> Symbol.TYPE, None) if dep_type is None or dep_type == <str> Symbol.ALL: dep_type = None elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]: @@ -94,9 +94,9 @@ cdef class Dependency: "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'" .format(provenance, dep_type)) - self.name = <str> _yaml.node_get(<_yaml.Node> dep, str, <str> Symbol.FILENAME) + self.name = (<_yaml.MappingNode> dep).get_str(<str> Symbol.FILENAME) self.dep_type = dep_type - self.junction = <str> _yaml.node_get(<_yaml.Node> dep, str, <str> Symbol.JUNCTION, None, None) + self.junction = (<_yaml.MappingNode> dep).get_str(<str> Symbol.JUNCTION, None) else: raise LoadError(LoadErrorReason.INVALID_DATA, @@ -138,15 +138,15 @@ cdef class Dependency: # acc (list): a list in which to add the loaded dependencies # cdef void _extract_depends_from_node(_yaml.Node node, str key, str default_dep_type, list acc) except *: - cdef list depends = <list> _yaml.node_get(node, list, key, None, []) - cdef int index + cdef _yaml.SequenceNode depends = node.get_sequence(key, []) + cdef _yaml.Node dep_node cdef _yaml.ProvenanceInformation dep_provenance - for index in range(len(depends)): + for dep_node in depends: # FIXME: the provenance information would be obtainable from the Node directly if we stop # stripping provenance and have proper nodes for str elements - dep_provenance = <_yaml.ProvenanceInformation> _yaml.node_get_provenance(node, key=key, indices=[index]) - dependency = Dependency(depends[index], dep_provenance, default_dep_type=default_dep_type) + dep_provenance = <_yaml.ProvenanceInformation> _yaml.node_get_provenance(dep_node) + dependency = Dependency(dep_node, dep_provenance, default_dep_type=default_dep_type) acc.append(dependency) # Now delete the field, we dont want it anymore diff --git a/src/buildstream/_options/option.py b/src/buildstream/_options/option.py index ffdb4d272..6aa4cdc01 100644 --- a/src/buildstream/_options/option.py +++ b/src/buildstream/_options/option.py @@ -60,8 +60,9 @@ class Option(): # node (dict): The loaded YAML dictionary describing # the option def load(self, node): - self.description = _yaml.node_get(node, str, 'description') - self.variable = _yaml.node_get(node, str, 'variable', default_value=None) + self.description = node.get_str('description') + variable_node = node.get_str('variable', default=None) + self.variable = variable_node # Assert valid symbol name for variable name if self.variable is not None: diff --git a/src/buildstream/_options/optionbool.py b/src/buildstream/_options/optionbool.py index bdbb1d32a..1e5935e48 100644 --- a/src/buildstream/_options/optionbool.py +++ b/src/buildstream/_options/optionbool.py @@ -34,13 +34,13 @@ class OptionBool(Option): super().load(node) _yaml.node_validate(node, OPTION_SYMBOLS + ['default']) - self.value = _yaml.node_get(node, bool, 'default') + self.value = node.get_bool('default') def load_value(self, node, *, transform=None): if transform: - self.set_value(transform(_yaml.node_get(node, str, self.name))) + self.set_value(transform(node.get_str(self.name))) else: - self.value = _yaml.node_get(node, bool, self.name) + self.value = node.get_bool(self.name) def set_value(self, value): if value in ('True', 'true'): diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py index 889db965c..f04cecd8b 100644 --- a/src/buildstream/_options/optionenum.py +++ b/src/buildstream/_options/optionenum.py @@ -46,7 +46,7 @@ class OptionEnum(Option): _yaml.node_validate(node, valid_symbols) - self.values = _yaml.node_get(node, list, 'values', default_value=[]) + self.values = node.get_sequence('values', default=[]).as_str_list() if not self.values: raise LoadError(LoadErrorReason.INVALID_DATA, "{}: No values specified for {} option '{}'" @@ -56,7 +56,7 @@ class OptionEnum(Option): self.value = self.load_default_value(node) def load_value(self, node, *, transform=None): - self.value = _yaml.node_get(node, str, self.name) + self.value = node.get_str(self.name) if transform: self.value = transform(self.value) self.validate(self.value, _yaml.node_get_provenance(node, self.name)) @@ -79,6 +79,6 @@ class OptionEnum(Option): "Valid values: {}".format(", ".join(self.values))) def load_default_value(self, node): - value = _yaml.node_get(node, str, 'default') + value = node.get_str('default') self.validate(value, _yaml.node_get_provenance(node, 'default')) return value diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py index eba3a8dd5..c9758e403 100644 --- a/src/buildstream/_options/optionflags.py +++ b/src/buildstream/_options/optionflags.py @@ -53,11 +53,11 @@ class OptionFlags(Option): "{}: No values specified for {} option '{}'" .format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name)) - self.value = _yaml.node_get(node, list, 'default', default_value=[]) + self.value = node.get_sequence('default', default=[]).as_str_list() self.validate(self.value, _yaml.node_get_provenance(node, 'default')) def load_value(self, node, *, transform=None): - self.value = _yaml.node_get(node, list, self.name) + self.value = node.get_sequence(self.name).as_str_list() if transform: self.value = [transform(x) for x in self.value] self.value = sorted(self.value) @@ -90,4 +90,4 @@ class OptionFlags(Option): def load_valid_values(self, node): # Allow the more descriptive error to raise when no values # exist rather than bailing out here (by specifying default_value) - return _yaml.node_get(node, list, 'values', default_value=[]) + return node.get_sequence('values', default=[]).as_str_list() diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py index de3af3e15..9a4a88c36 100644 --- a/src/buildstream/_options/optionpool.py +++ b/src/buildstream/_options/optionpool.py @@ -71,7 +71,7 @@ class OptionPool(): p = _yaml.node_get_provenance(options, option_name) _yaml.assert_symbol_name(p, option_name, "option name", allow_dashes=False) - opt_type_name = _yaml.node_get(option_definition, str, 'type') + opt_type_name = option_definition.get_str('type') try: opt_type = _OPTION_TYPES[opt_type_name] except KeyError: @@ -248,8 +248,8 @@ class OptionPool(): # Return true if a conditional was processed. # def _process_one_node(self, node): - conditions = _yaml.node_get(node, list, '(?)', default_value=None) - assertion = _yaml.node_get(node, str, '(!)', default_value=None) + conditions = node.get_sequence('(?)', default=None) + assertion = node.get_str('(!)', default=None) # Process assersions first, we want to abort on the first encountered # assertion in a given dictionary, and not lose an assertion due to diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py index 7a5407cf6..e95e881c4 100644 --- a/src/buildstream/_plugincontext.py +++ b/src/buildstream/_plugincontext.py @@ -22,7 +22,6 @@ import inspect from ._exceptions import PluginError, LoadError, LoadErrorReason from . import utils -from . import _yaml # A Context for loading plugin types @@ -138,19 +137,19 @@ class PluginContext(): loaded_dependency = False for origin in self._plugin_origins: - if kind not in _yaml.node_get(origin, list, 'plugins'): + if kind not in origin.get_sequence('plugins').as_str_list(): continue - if _yaml.node_get(origin, str, 'origin') == 'local': - local_path = _yaml.node_get(origin, str, 'path') + if origin.get_str('origin') == 'local': + local_path = origin.get_str('path') source = self._get_local_plugin_source(local_path) - elif _yaml.node_get(origin, str, 'origin') == 'pip': - package_name = _yaml.node_get(origin, str, 'package-name') + elif origin.get_str('origin') == 'pip': + package_name = origin.get_str('package-name') source, defaults = self._get_pip_plugin_source(package_name, kind) else: raise PluginError("Failed to load plugin '{}': " "Unexpected plugin origin '{}'" - .format(kind, _yaml.node_get(origin, str, 'origin'))) + .format(kind, origin.get_str('origin'))) loaded_dependency = True break diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 114d25054..424af133e 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -84,7 +84,7 @@ class ProjectConfig: self.source_overrides = {} # Source specific configurations self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs. self.default_mirror = None # The name of the preferred mirror. - self._aliases = {} # Aliases dictionary + self._aliases = None # Aliases dictionary # Project() @@ -200,7 +200,7 @@ class Project(): if url and utils._ALIAS_SEPARATOR in url: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) - alias_url = _yaml.node_get(config._aliases, str, url_alias, default_value=None) + alias_url = config._aliases.get_str(url_alias, default=None) if alias_url: url = alias_url + url_body @@ -264,7 +264,7 @@ class Project(): # def get_path_from_node(self, node, key, *, check_is_file=False, check_is_dir=False): - path_str = _yaml.node_get(node, str, key) + path_str = node.get_str(key) path = Path(path_str) full_path = self._absolute_directory_path / path @@ -404,7 +404,7 @@ class Project(): else: config = self.config - return _yaml.node_get(config._aliases, str, alias, default_value=None) + return config._aliases.get_str(alias, default=None) # get_alias_uris() # @@ -419,7 +419,7 @@ class Project(): else: config = self.config - if not alias or alias not in config._aliases: + if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test return [None] mirror_list = [] @@ -429,7 +429,7 @@ class Project(): mirror_list = alias_mapping[alias] + mirror_list else: mirror_list += alias_mapping[alias] - mirror_list.append(_yaml.node_get(config._aliases, str, alias)) + mirror_list.append(config._aliases.get_str(alias)) return mirror_list # load_elements() @@ -573,7 +573,7 @@ class Project(): _yaml.composite(pre_config_node, self._project_conf) # Assert project's format version early, before validating toplevel keys - format_version = _yaml.node_get(pre_config_node, int, 'format-version') + format_version = pre_config_node.get_int('format-version') if BST_FORMAT_VERSION < format_version: major, minor = utils.get_bst_version() raise LoadError( @@ -585,7 +585,7 @@ class Project(): # The project name, element path and option declarations # are constant and cannot be overridden by option conditional statements - self.name = _yaml.node_get(self._project_conf, str, 'name') + self.name = self._project_conf.get_str('name') # Validate that project name is a valid symbol name _yaml.assert_symbol_name(_yaml.node_get_provenance(pre_config_node, 'name'), @@ -600,12 +600,12 @@ class Project(): self.config.options = OptionPool(self.element_path) self.first_pass_config.options = OptionPool(self.element_path) - defaults = _yaml.node_get(pre_config_node, dict, 'defaults') + defaults = pre_config_node.get_mapping('defaults') _yaml.node_validate(defaults, ['targets']) - self._default_targets = _yaml.node_get(defaults, list, "targets") + self._default_targets = defaults.get_sequence("targets").as_str_list() # Fatal warnings - self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[]) + self._fatal_warnings = pre_config_node.get_sequence('fatal-warnings', default=[]).as_str_list() self.loader = Loader(self._context, self, parent=parent_loader, fetch_subprojects=fetch_subprojects) @@ -621,7 +621,7 @@ class Project(): ignore_unknown=True) # Use separate file for storing source references - self.ref_storage = _yaml.node_get(pre_config_node, str, 'ref-storage') + self.ref_storage = pre_config_node.get_str('ref-storage') if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]: p = _yaml.node_get_provenance(pre_config_node, 'ref-storage') raise LoadError(LoadErrorReason.INVALID_DATA, @@ -677,23 +677,20 @@ class Project(): self.remote_execution_specs = self._context.remote_execution_specs # Load sandbox environment variables - self.base_environment = _yaml.node_get(config, dict, 'environment') - self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache') + self.base_environment = config.get_mapping('environment') + self.base_env_nocache = config.get_sequence('environment-nocache').as_str_list() # Load sandbox configuration - self._sandbox = _yaml.node_get(config, dict, 'sandbox') + self._sandbox = config.get_mapping('sandbox') # Load project split rules - self._splits = _yaml.node_get(config, dict, 'split-rules') + self._splits = config.get_mapping('split-rules') # Support backwards compatibility for fail-on-overlap - fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None) - - if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap: - self._fatal_warnings.append(CoreWarnings.OVERLAPS) + fail_on_overlap = config.get_scalar('fail-on-overlap', None) # Deprecation check - if fail_on_overlap is not None: + if not fail_on_overlap.is_none(): self._context.message( Message( None, @@ -703,36 +700,37 @@ class Project(): ) ) + if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap.as_bool(): + self._fatal_warnings.append(CoreWarnings.OVERLAPS) + # Load project.refs if it exists, this may be ignored. if self.ref_storage == ProjectRefStorage.PROJECT_REFS: self.refs.load(self.options) # Parse shell options - shell_options = _yaml.node_get(config, dict, 'shell') + shell_options = config.get_mapping('shell') _yaml.node_validate(shell_options, ['command', 'environment', 'host-files']) - self._shell_command = _yaml.node_get(shell_options, list, 'command') + self._shell_command = shell_options.get_sequence('command').as_str_list() # Perform environment expansion right away - shell_environment = _yaml.node_get(shell_options, dict, 'environment', default_value={}) + shell_environment = shell_options.get_mapping('environment', default={}) for key in _yaml.node_keys(shell_environment): - value = _yaml.node_get(shell_environment, str, key) + value = shell_environment.get_str(key) self._shell_environment[key] = os.path.expandvars(value) # Host files is parsed as a list for convenience - host_files = _yaml.node_get(shell_options, list, 'host-files', default_value=[]) + host_files = shell_options.get_sequence('host-files', default=[]) for host_file in host_files: - if isinstance(host_file, str): + if isinstance(host_file, _yaml.ScalarNode): mount = HostMount(host_file) else: # Some validation - index = host_files.index(host_file) - host_file_desc = _yaml.node_get(shell_options, dict, 'host-files', indices=[index]) - _yaml.node_validate(host_file_desc, ['path', 'host_path', 'optional']) + _yaml.node_validate(host_file, ['path', 'host_path', 'optional']) # Parse the host mount - path = _yaml.node_get(host_file_desc, str, 'path') - host_path = _yaml.node_get(host_file_desc, str, 'host_path', default_value=None) - optional = _yaml.node_get(host_file_desc, bool, 'optional', default_value=False) + path = host_file.get_str('path') + host_path = host_file.get_str('host_path', default=None) + optional = host_file.get_bool('optional', default=False) mount = HostMount(path, host_path, optional) self._shell_host_files.append(mount) @@ -753,8 +751,8 @@ class Project(): # Element and Source type configurations will be composited later onto # element/source types, so we delete it from here and run our final # assertion after. - output.element_overrides = _yaml.node_get(config, dict, 'elements', default_value={}) - output.source_overrides = _yaml.node_get(config, dict, 'sources', default_value={}) + output.element_overrides = config.get_mapping('elements', default={}) + output.source_overrides = config.get_mapping('sources', default={}) _yaml.node_del(config, 'elements', safe=True) _yaml.node_del(config, 'sources', safe=True) _yaml.node_final_assertions(config) @@ -762,7 +760,7 @@ class Project(): self._load_plugin_factories(config, output) # Load project options - options_node = _yaml.node_get(config, dict, 'options', default_value={}) + options_node = config.get_mapping('options', default={}) output.options.load(options_node) if self.junction: # load before user configuration @@ -770,7 +768,7 @@ class Project(): # Collect option values specified in the user configuration overrides = self._context.get_overrides(self.name) - override_options = _yaml.node_get(overrides, dict, 'options', default_value={}) + override_options = overrides.get_mapping('options', default={}) output.options.load_yaml_values(override_options) if self._cli_options: output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown) @@ -789,7 +787,7 @@ class Project(): output.options.process_node(output.source_overrides) # Load base variables - output.base_variables = _yaml.node_get(config, dict, 'variables') + output.base_variables = config.get_mapping('variables') # Add the project name as a default variable _yaml.node_set(output.base_variables, 'project-name', self.name) @@ -806,18 +804,18 @@ class Project(): output.options.export_variables(output.base_variables) # Override default_mirror if not set by command-line - output.default_mirror = self._default_mirror or _yaml.node_get(overrides, str, - 'default-mirror', default_value=None) + output.default_mirror = self._default_mirror or overrides.get_str( + 'default-mirror', default=None) - mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[]) + mirrors = config.get_sequence('mirrors', default=[]) for mirror in mirrors: allowed_mirror_fields = [ 'name', 'aliases' ] _yaml.node_validate(mirror, allowed_mirror_fields) - mirror_name = _yaml.node_get(mirror, str, 'name') + mirror_name = mirror.get_str('name') alias_mappings = {} - for alias_mapping, uris in _yaml.node_items(_yaml.node_get(mirror, dict, 'aliases')): + for alias_mapping, uris in _yaml.node_items(mirror.get_mapping('aliases')): assert isinstance(uris, list) alias_mappings[alias_mapping] = list(uris) output.mirrors[mirror_name] = alias_mappings @@ -825,7 +823,7 @@ class Project(): output.default_mirror = mirror_name # Source url aliases - output._aliases = _yaml.node_get(config, dict, 'aliases', default_value={}) + output._aliases = config.get_mapping('aliases', default={}) # _find_project_dir() # @@ -869,7 +867,7 @@ class Project(): plugin_element_origins = [] # Origins of custom elements # Plugin origins and versions - origins = _yaml.node_get(config, list, 'plugins', default_value=[]) + origins = config.get_sequence('plugins', default=[]) source_format_versions = {} element_format_versions = {} for origin in origins: @@ -880,7 +878,7 @@ class Project(): allowed_origins = ['core', 'local', 'pip'] _yaml.node_validate(origin, allowed_origin_fields) - origin_value = _yaml.node_get(origin, str, 'origin') + origin_value = origin.get_str('origin') if origin_value not in allowed_origins: raise LoadError( LoadErrorReason.INVALID_YAML, @@ -888,26 +886,26 @@ class Project(): .format(origin_value)) # Store source versions for checking later - source_versions = _yaml.node_get(origin, dict, 'sources', default_value={}) + source_versions = origin.get_mapping('sources', default={}) for key in _yaml.node_keys(source_versions): if key in source_format_versions: raise LoadError( LoadErrorReason.INVALID_YAML, "Duplicate listing of source '{}'".format(key)) - source_format_versions[key] = _yaml.node_get(source_versions, int, key) + source_format_versions[key] = source_versions.get_int(key) # Store element versions for checking later - element_versions = _yaml.node_get(origin, dict, 'elements', default_value={}) + element_versions = origin.get_mapping('elements', default={}) for key in _yaml.node_keys(element_versions): if key in element_format_versions: raise LoadError( LoadErrorReason.INVALID_YAML, "Duplicate listing of element '{}'".format(key)) - element_format_versions[key] = _yaml.node_get(element_versions, int, key) + element_format_versions[key] = element_versions.get_int(key) # Store the origins if they're not 'core'. # core elements are loaded by default, so storing is unnecessary. - if _yaml.node_get(origin, str, 'origin') != 'core': + if origin.get_str('origin') != 'core': self._store_origin(origin, 'sources', plugin_source_origins) self._store_origin(origin, 'elements', plugin_element_origins) @@ -941,13 +939,13 @@ class Project(): node_keys = [key for key in _yaml.node_keys(origin)] if plugin_group in node_keys: origin_node = _yaml.node_copy(origin) - plugins = _yaml.node_get(origin, dict, plugin_group, default_value={}) + plugins = origin.get_mapping(plugin_group, default={}) _yaml.node_set(origin_node, 'plugins', [k for k in _yaml.node_keys(plugins)]) for group in expected_groups: if group in origin_node: _yaml.node_del(origin_node, group) - if _yaml.node_get(origin_node, str, 'origin') == 'local': + if origin_node.get_str('origin') == 'local': path = self.get_path_from_node(origin, 'path', check_is_dir=True) # paths are passed in relative to the project, but must be absolute diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py index 09205a7c3..e72d4757e 100644 --- a/src/buildstream/_projectrefs.py +++ b/src/buildstream/_projectrefs.py @@ -123,8 +123,8 @@ class ProjectRefs(): def _lookup(self, toplevel, project, element, source_index, *, ensure=False): # Fetch the project try: - projects = _yaml.node_get(toplevel, dict, 'projects') - project_node = _yaml.node_get(projects, dict, project) + projects = toplevel.get_mapping('projects') + project_node = projects.get_mapping(project) except LoadError: if not ensure: return None @@ -133,16 +133,16 @@ class ProjectRefs(): # Fetch the element try: - element_list = _yaml.node_get(project_node, list, element) + element_list = project_node.get_sequence(element) except LoadError: if not ensure: return None - element_list = [] + element_list = _yaml.new_empty_list_node() _yaml.node_set(project_node, element, element_list) # Fetch the source index try: - node = element_list[source_index] + node = element_list.mapping_at(source_index) except IndexError: if not ensure: return None @@ -150,6 +150,6 @@ class ProjectRefs(): # Pad the list with empty newly created dictionaries _yaml.node_extend_list(project_node, element, source_index + 1, {}) - node = _yaml.node_get(project_node, dict, element, indices=[source_index]) + node = project_node.get_sequence(element).mapping_at(source_index) return node diff --git a/src/buildstream/_variables.pyx b/src/buildstream/_variables.pyx index 9b8b5a902..e8f083912 100644 --- a/src/buildstream/_variables.pyx +++ b/src/buildstream/_variables.pyx @@ -120,7 +120,7 @@ cdef class Variables: # element, then override max-jobs to be 1. # Initialize it as a string as all variables are processed as strings. # - if _yaml.node_get(node, bool, 'notparallel', None, False): + if node.get_bool('notparallel', False): _yaml.node_set(node, 'max-jobs', str(1)) cdef dict ret = {} @@ -128,7 +128,7 @@ cdef class Variables: cdef str value for key in _yaml.node_keys(node): - value = <str> _yaml.node_get(node, str, key) + value = node.get_str(key) ret[sys.intern(key)] = _parse_expstr(value) return ret diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py index 9fbfb7e63..3847c7a85 100644 --- a/src/buildstream/_workspaces.py +++ b/src/buildstream/_workspaces.py @@ -570,7 +570,7 @@ class Workspaces(): # def _parse_workspace_config(self, workspaces): try: - version = _yaml.node_get(workspaces, int, 'format-version', default_value=0) + version = workspaces.get_int('format-version', default=0) except ValueError: raise LoadError(LoadErrorReason.INVALID_DATA, "Format version is not an integer in workspace configuration") @@ -606,8 +606,7 @@ class Workspaces(): } elif 1 <= version <= BST_WORKSPACE_FORMAT_VERSION: - workspaces = _yaml.node_get(workspaces, dict, "workspaces", - default_value=_yaml.new_empty_node()) + workspaces = workspaces.get_mapping("workspaces", default={}) res = {element: self._load_workspace(node) for element, node in _yaml.node_items(workspaces)} @@ -631,11 +630,11 @@ class Workspaces(): # def _load_workspace(self, node): dictionary = { - 'prepared': _yaml.node_get(node, bool, 'prepared', default_value=False), - 'path': _yaml.node_get(node, str, 'path'), - 'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None), + 'prepared': node.get_bool('prepared', default=False), + 'path': node.get_str('path'), + 'last_successful': node.get_str('last_successful', default=None), 'running_files': _yaml.node_sanitize( - _yaml.node_get(node, dict, 'running_files', default_value=None), + node.get_mapping('running_files', default=None), dict_type=dict), } return Workspace.from_dict(self._toplevel_project, dictionary) diff --git a/src/buildstream/_yaml.pxd b/src/buildstream/_yaml.pxd index 3cbad0d11..8deb17a78 100644 --- a/src/buildstream/_yaml.pxd +++ b/src/buildstream/_yaml.pxd @@ -28,6 +28,30 @@ cdef class Node: cdef public int column +cdef class MappingNode(Node): + cdef Node get(self, str key, default, default_constructor) + cpdef MappingNode get_mapping(self, str key, default=*) + cpdef Node get_node(self, str key, list allowed_types, bint allow_none=*) + cpdef ScalarNode get_scalar(self, str key, default=*) + cpdef SequenceNode get_sequence(self, str key, object default=*) + cpdef bint get_bool(self, str key, default=*) except * + cpdef int get_int(self, str key, default=*) except * + cpdef str get_str(self, str key, object default=*) + + +cdef class ScalarNode(Node): + cpdef bint as_bool(self) except * + cpdef int as_int(self) except * + cpdef str as_str(self) + cpdef bint is_none(self) + + +cdef class SequenceNode(Node): + cpdef MappingNode mapping_at(self, int index) + cpdef SequenceNode sequence_at(self, int index) + cpdef list as_str_list(self) + + cdef class ProvenanceInformation: cdef public Node node @@ -39,7 +63,6 @@ cdef class ProvenanceInformation: cpdef void node_del(Node node, str key, bint safe=*) except * -cpdef object node_get(Node node, object expected_type, str key, list indices=*, object default_value=*, bint allow_none=*) cpdef void node_validate(Node node, list valid_keys) except * cpdef void node_set(Node node, object key, object value, list indices=*) except * cpdef list node_keys(Node node) diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx index 678374272..4a49a9832 100644 --- a/src/buildstream/_yaml.pyx +++ b/src/buildstream/_yaml.pyx @@ -40,6 +40,11 @@ from ._exceptions import LoadError, LoadErrorReason # pylint: disable=unidiomatic-typecheck +# A sentinel to be used as a default argument for functions that need +# to distinguish between a kwarg set to None and an unset kwarg. +_sentinel = object() + + # Node() # # Container for YAML loaded data and its provenance @@ -70,6 +75,190 @@ cdef class Node: return what in self.value +cdef class ScalarNode(Node): + + def __init__(self, object value, int file_index, int line, int column): + if type(value) is str: + value = value.strip() + self.value = value + self.file_index = file_index + self.line = line + self.column = column + + cpdef bint is_none(self): + return self.value is None + + cpdef bint as_bool(self) except *: + if type(self.value) is bool: + return self.value + + # Don't coerce booleans to string, this makes "False" strings evaluate to True + if self.value in ('True', 'true'): + return True + elif self.value in ('False', 'false'): + return False + else: + provenance = node_get_provenance(self) + path = node_find_target(provenance.toplevel, self)[-1] + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type '{}'" + .format(provenance, path, bool.__name__, self.value)) + + cpdef int as_int(self) except *: + try: + return int(self.value) + except ValueError: + provenance = node_get_provenance(self) + path = node_find_target(provenance.toplevel, self)[-1] + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type '{}'" + .format(provenance, path, int.__name__)) + + cpdef str as_str(self): + # We keep 'None' as 'None' to simplify the API's usage and allow chaining for users + if self.value is None: + return None + return str(self.value) + + +cdef class MappingNode(Node): + + def __init__(self, dict value, int file_index, int line, int column): + self.value = value + self.file_index = file_index + self.line = line + self.column = column + + cdef Node get(self, str key, object default, object default_constructor): + value = self.value.get(key, _sentinel) + + if value is _sentinel: + if default is _sentinel: + provenance = node_get_provenance(self) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Dictionary did not contain expected key '{}'".format(provenance, key)) + + if default is None: + value = None + else: + value = default_constructor(default, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) + + return value + + cpdef MappingNode get_mapping(self, str key, object default=_sentinel): + value = self.get(key, default, MappingNode) + + if type(value) is not MappingNode and value is not None: + provenance = node_get_provenance(value) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type 'Mapping'" + .format(provenance, key)) + + return value + + cpdef Node get_node(self, str key, list allowed_types, bint allow_none = False): + cdef value = self.value.get(key, _sentinel) + + if value is _sentinel: + if allow_none: + return None + + provenance = node_get_provenance(self) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Dictionary did not contain expected key '{}'".format(provenance, key)) + + if type(value) not in allowed_types: + provenance = node_get_provenance(self) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not one of the following: {}.".format( + provenance, key, ", ".join(allowed_types))) + + return value + + cpdef ScalarNode get_scalar(self, str key, object default=_sentinel): + value = self.get(key, default, ScalarNode) + + if type(value) is not ScalarNode: + if value is None: + value = ScalarNode(None, self.file_index, 0, next_synthetic_counter()) + else: + provenance = node_get_provenance(value) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type 'Scalar'" + .format(provenance, key)) + + return value + + cpdef SequenceNode get_sequence(self, str key, object default=_sentinel): + value = self.get(key, default, SequenceNode) + + if type(value) is not SequenceNode and value is not None: + provenance = node_get_provenance(value) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type 'Sequence'" + .format(provenance, key)) + + return value + + cpdef bint get_bool(self, str key, object default=_sentinel) except *: + # TODO: don't go through 'get_scalar', we can directly get everything and optimize + cdef ScalarNode scalar = self.get_scalar(key, default) + return scalar.as_bool() + + cpdef int get_int(self, str key, object default=_sentinel) except *: + # TODO: don't go through 'get_scalar', we can directly get everything and optimize + cdef ScalarNode scalar = self.get_scalar(key, default) + return scalar.as_int() + + cpdef str get_str(self, str key, object default=_sentinel): + # TODO: don't go through 'get_scalar', we can directly get everything and optimize + cdef ScalarNode scalar = self.get_scalar(key, default) + return scalar.as_str() + + +cdef class SequenceNode(Node): + def __init__(self, list value, int file_index, int line, int column): + self.value = value + self.file_index = file_index + self.line = line + self.column = column + + cpdef MappingNode mapping_at(self, int index): + value = self.value[index] + + if type(value) is not MappingNode: + provenance = node_get_provenance(self) + path = ["[{}]".format(p) for p in node_find_target(provenance.toplevel, self)] + ["[{}]".format(index)] + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type '{}'" + .format(provenance, path, MappingNode.__name__)) + return value + + cpdef SequenceNode sequence_at(self, int index): + value = self.value[index] + + if type(value) is not SequenceNode: + provenance = node_get_provenance(self) + path = ["[{}]".format(p) for p in node_find_target(provenance.toplevel, self)] + ["[{}]".format(index)] + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Value of '{}' is not of the expected type '{}'" + .format(provenance, path, SequenceNode.__name__)) + + return value + + cpdef list as_str_list(self): + return [node.as_str() for node in self.value] + + def __iter__(self): + return iter(self.value) + + def __len__(self): + return len(self.value) + + def __reversed__(self): + return reversed(self.value) + + # Metadata container for a yaml toplevel node. # # This class contains metadata around a yaml node in order to be able @@ -280,7 +469,7 @@ cdef class Representer: return RepresenterState.doc cdef RepresenterState _handle_doc_MappingStartEvent(self, object ev): - newmap = Node({}, self._file_index, ev.start_mark.line, ev.start_mark.column) + newmap = MappingNode({}, self._file_index, ev.start_mark.line, ev.start_mark.column) self.output.append(newmap) return RepresenterState.wait_key @@ -291,13 +480,13 @@ cdef class Representer: cdef RepresenterState _handle_wait_value_ScalarEvent(self, object ev): key = self.keys.pop() (<dict> (<Node> self.output[-1]).value)[key] = \ - Node(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column) + ScalarNode(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column) return RepresenterState.wait_key cdef RepresenterState _handle_wait_value_MappingStartEvent(self, object ev): cdef RepresenterState new_state = self._handle_doc_MappingStartEvent(ev) key = self.keys.pop() - (<dict> (<Node> self.output[-2]).value)[key] = self.output[-1] + (<dict> (<MappingNode> self.output[-2]).value)[key] = self.output[-1] return new_state cdef RepresenterState _handle_wait_key_MappingEndEvent(self, object ev): @@ -313,13 +502,13 @@ cdef class Representer: return RepresenterState.doc cdef RepresenterState _handle_wait_value_SequenceStartEvent(self, object ev): - self.output.append(Node([], self._file_index, ev.start_mark.line, ev.start_mark.column)) + self.output.append(SequenceNode([], self._file_index, ev.start_mark.line, ev.start_mark.column)) (<dict> (<Node> self.output[-2]).value)[self.keys[-1]] = self.output[-1] return RepresenterState.wait_list_item cdef RepresenterState _handle_wait_list_item_SequenceStartEvent(self, object ev): self.keys.append(len((<Node> self.output[-1]).value)) - self.output.append(Node([], self._file_index, ev.start_mark.line, ev.start_mark.column)) + self.output.append(SequenceNode([], self._file_index, ev.start_mark.line, ev.start_mark.column)) (<list> (<Node> self.output[-2]).value).append(self.output[-1]) return RepresenterState.wait_list_item @@ -336,7 +525,7 @@ cdef class Representer: cdef RepresenterState _handle_wait_list_item_ScalarEvent(self, object ev): (<Node> self.output[-1]).value.append( - Node(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column)) + ScalarNode(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column)) return RepresenterState.wait_list_item cdef RepresenterState _handle_wait_list_item_MappingStartEvent(self, object ev): @@ -353,6 +542,19 @@ cdef class Representer: return RepresenterState.init +cdef Node _create_node(object value, int file_index, int line, int column): + cdef type_value = type(value) + + if type_value in [bool, str, type(None), int]: + return ScalarNode(value, file_index, line, column) + elif type_value is dict: + return MappingNode(value, file_index, line, column) + elif type_value is list: + return SequenceNode(value, file_index, line, column) + raise ValueError( + "Node values can only be 'list', 'dict', 'bool', 'str', 'int' or None. Not {}".format(type_value)) + + # Loads a dictionary from some YAML # # Args: @@ -426,10 +628,10 @@ cpdef Node load_data(str data, int file_index=_SYNTHETIC_FILE_INDEX, str file_na raise LoadError(LoadErrorReason.INVALID_YAML, "Severely malformed YAML:\n\n{}\n\n".format(e)) from e - if type(contents) != Node: + if type(contents) != MappingNode: # Special case allowance for None, when the loaded file has only comments in it. if contents is None: - contents = Node({}, file_index, 0, 0) + contents = MappingNode({}, file_index, 0, 0) else: raise LoadError(LoadErrorReason.INVALID_YAML, "YAML file has content of type '{}' instead of expected type 'dict': {}" @@ -479,8 +681,6 @@ def dump(object contents, str filename=None): # Returns: The Provenance of the dict, member or list element # cpdef ProvenanceInformation node_get_provenance(Node node, str key=None, list indices=None): - assert type(node.value) is dict - if key is None: # Retrieving the provenance for this node directly return ProvenanceInformation(node) @@ -495,105 +695,6 @@ cpdef ProvenanceInformation node_get_provenance(Node node, str key=None, list in return ProvenanceInformation(nodeish) -# A sentinel to be used as a default argument for functions that need -# to distinguish between a kwarg set to None and an unset kwarg. -_sentinel = object() - - -# node_get() -# -# Fetches a value from a dictionary node and checks it for -# an expected value. Use default_value when parsing a value -# which is only optionally supplied. -# -# Args: -# node (dict): The dictionary node -# expected_type (type): The expected type for the value being searched -# key (str): The key to get a value for in node -# indices (list of ints): Optionally decend into lists of lists -# default_value: Optionally return this value if the key is not found -# allow_none: (bool): Allow None to be a valid value -# -# Returns: -# The value if found in node, otherwise default_value is returned -# -# Raises: -# LoadError, when the value found is not of the expected type -# -# Note: -# Returned strings are stripped of leading and trailing whitespace -# -cpdef object node_get(Node node, object expected_type, str key, list indices=None, object default_value=_sentinel, bint allow_none=False): - if indices is None: - value = node.value.get(key, _sentinel) - - if value is _sentinel: - if default_value is _sentinel: - provenance = node_get_provenance(node) - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: Dictionary did not contain expected key '{}'".format(provenance, key)) - - value = Node(default_value, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) - else: - # Implied type check of the element itself - # No need to synthesise useful node content as we destructure it immediately - value = Node(node_get(node, list, key), _SYNTHETIC_FILE_INDEX, 0, 0) - for index in indices: - value = value.value[index] - if type(value) is not Node: - value = Node(value, _SYNTHETIC_FILE_INDEX, 0, 0) - - # Optionally allow None as a valid value for any type - if value.value is None and (allow_none or default_value is None): - return None - - if (expected_type is not None) and (type(value.value) is not expected_type): - # Attempt basic conversions if possible, typically we want to - # be able to specify numeric values and convert them to strings, - # but we dont want to try converting dicts/lists - try: - if expected_type == bool and type(value.value) is str: - # Dont coerce booleans to string, this makes "False" strings evaluate to True - # We don't structure into full nodes since there's no need. - if value.value in ('True', 'true'): - value = Node(True, _SYNTHETIC_FILE_INDEX, 0, 0) - elif value.value in ('False', 'false'): - value = Node(False, _SYNTHETIC_FILE_INDEX, 0, 0) - else: - raise ValueError() - elif not (expected_type == list or - expected_type == dict or - isinstance(value.value, (list, dict))): - value = Node(expected_type(value.value), _SYNTHETIC_FILE_INDEX, 0, 0) - else: - raise ValueError() - except (ValueError, TypeError): - provenance = node_get_provenance(node, key=key, indices=indices) - if indices: - path = [key, *["[{:d}]".format(i) for i in indices]] - path = "".join(path) - else: - path = key - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: Value of '{}' is not of the expected type '{}'" - .format(provenance, path, expected_type.__name__)) - - # Now collapse lists, and scalars, to their value, leaving nodes as-is - if type(value.value) is not dict: - value = value.value - - # Trim it at the bud, let all loaded strings from yaml be stripped of whitespace - if type(value) is str: - value = value.strip() - - elif type(value) is list: - # Now we create a fresh list which unwraps the str and list types - # semi-recursively. - value = __trim_list_provenance(value) - - return value - - cdef list __trim_list_provenance(list value): cdef list ret = [] cdef Node entry @@ -632,7 +733,7 @@ cpdef void node_set(Node node, object key, object value, list indices=None) exce key = indices.pop() for idx in indices: node = <Node> (<list> node.value)[idx] - if type(value) is Node: + if type(value) in [Node, MappingNode, ScalarNode, SequenceNode]: node.value[key] = value else: try: @@ -640,10 +741,11 @@ cpdef void node_set(Node node, object key, object value, list indices=None) exce old_value = <Node> node.value[key] except KeyError: old_value = None + if old_value is None: - node.value[key] = Node(value, node.file_index, node.line, next_synthetic_counter()) + node.value[key] = _create_node(value, node.file_index, node.line, next_synthetic_counter()) else: - node.value[key] = Node(value, old_value.file_index, old_value.line, old_value.column) + node.value[key] = _create_node(value, old_value.file_index, old_value.line, old_value.column) # node_extend_list() @@ -666,7 +768,7 @@ def node_extend_list(Node node, str key, Py_ssize_t length, object default): cdef Node list_node = <Node> node.value.get(key) if list_node is None: - list_node = node.value[key] = Node([], node.file_index, node.line, next_synthetic_counter()) + list_node = node.value[key] = SequenceNode([], node.file_index, node.line, next_synthetic_counter()) cdef list the_list = list_node.value def_type = type(default) @@ -687,7 +789,7 @@ def node_extend_list(Node node, str key, Py_ssize_t length, object default): line_num += 1 - the_list.append(Node(value, file_index, line_num, next_synthetic_counter())) + the_list.append(_create_node(value, file_index, line_num, next_synthetic_counter())) # node_items() @@ -763,9 +865,9 @@ cpdef void node_del(Node node, str key, bint safe=False) except *: def is_node(maybenode): # It's a programming error to give this a Node which isn't a mapping # so assert that. - assert (type(maybenode) is not Node) or (type(maybenode.value) is dict) + assert (type(maybenode) not in [ScalarNode, SequenceNode]) # Now return the type check - return type(maybenode) is Node + return type(maybenode) is MappingNode # new_synthetic_file() @@ -784,7 +886,7 @@ def is_node(maybenode): # def new_synthetic_file(str filename, object project=None): cdef Py_ssize_t file_index = len(_FILE_LIST) - cdef Node node = Node({}, file_index, 0, 0) + cdef Node node = MappingNode({}, file_index, 0, 0) _FILE_LIST.append(FileInfo(filename, filename, @@ -804,9 +906,14 @@ def new_synthetic_file(str filename, object project=None): # def new_empty_node(Node ref_node=None): if ref_node is not None: - return Node({}, ref_node.file_index, ref_node.line, next_synthetic_counter()) + return MappingNode({}, ref_node.file_index, ref_node.line, next_synthetic_counter()) else: - return Node({}, _SYNTHETIC_FILE_INDEX, 0, 0) + return MappingNode({}, _SYNTHETIC_FILE_INDEX, 0, 0) + + +# FIXME: we should never need that +def new_empty_list_node(): + return SequenceNode([], _SYNTHETIC_FILE_INDEX, 0, 0) # new_node_from_dict() @@ -827,8 +934,8 @@ cpdef Node new_node_from_dict(dict indict): elif vtype is list: ret[k] = __new_node_from_list(v) else: - ret[k] = Node(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) - return Node(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) + ret[k] = ScalarNode(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) + return MappingNode(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) # Internal function to help new_node_from_dict() to handle lists @@ -841,8 +948,8 @@ cdef Node __new_node_from_list(list inlist): elif vtype is list: ret.append(__new_node_from_list(v)) else: - ret.append(Node(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())) - return Node(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) + ret.append(ScalarNode(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())) + return SequenceNode(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()) # _is_composite_list @@ -1000,7 +1107,7 @@ cpdef void composite_dict(Node target, Node source, list path=None) except *: if k not in target.value: # Target lacks a dict at that point, make a fresh one with # the same provenance as the incoming dict - target.value[k] = Node({}, v.file_index, v.line, v.column) + target.value[k] = MappingNode({}, v.file_index, v.line, v.column) if type(target.value) is not dict: raise CompositeError(path, "{}: Cannot compose dictionary onto {}".format( @@ -1020,7 +1127,7 @@ cpdef void composite_dict(Node target, Node source, list path=None) except *: # Like composite_dict(), but raises an all purpose LoadError for convenience # -cpdef void composite(Node target, Node source) except *: +cpdef void composite(MappingNode target, MappingNode source) except *: assert type(source.value) is dict assert type(target.value) is dict @@ -1040,7 +1147,7 @@ cpdef void composite(Node target, Node source) except *: # Like composite(target, source), but where target overrides source instead. # -def composite_and_move(Node target, Node source): +def composite_and_move(MappingNode target, MappingNode source): composite(source, target) cdef str key @@ -1067,10 +1174,14 @@ cpdef object node_sanitize(object node, object dict_type=OrderedDict): node_type = type(node) # If we have an unwrappable node, unwrap it - if node_type is Node: + # FIXME: we should only ever have Nodes here + if node_type in [MappingNode, SequenceNode]: node = node.value node_type = type(node) + if node_type is ScalarNode: + return node.value + # Short-circuit None which occurs ca. twice per element if node is None: return node @@ -1157,7 +1268,7 @@ __NODE_ASSERT_COMPOSITION_DIRECTIVES = ('(>)', '(<)', '(=)') # Returns: # (Node): A deep copy of source with provenance preserved. # -cpdef Node node_copy(Node source): +cpdef MappingNode node_copy(MappingNode source): cdef dict copy = {} cdef str key cdef Node value @@ -1173,7 +1284,7 @@ cpdef Node node_copy(Node source): else: raise ValueError("Unable to be quick about node_copy of {}".format(value_type)) - return Node(copy, source.file_index, source.line, source.column) + return MappingNode(copy, source.file_index, source.line, source.column) # Internal function to help node_copy() but for lists. @@ -1193,7 +1304,7 @@ cdef Node _list_copy(Node source): else: raise ValueError("Unable to be quick about list_copy of {}".format(item_type)) - return Node(copy, source.file_index, source.line, source.column) + return SequenceNode(copy, source.file_index, source.line, source.column) # node_final_assertions() @@ -1207,7 +1318,7 @@ cdef Node _list_copy(Node source): # Raises: # (LoadError): If any assertions fail # -cpdef void node_final_assertions(Node node) except *: +cpdef void node_final_assertions(MappingNode node) except *: cdef str key cdef Node value @@ -1303,7 +1414,7 @@ def assert_symbol_name(ProvenanceInformation provenance, str symbol_name, str pu # # Returns: # (list): A path from `node` to `target` or None if `target` is not in the subtree -cpdef list node_find_target(Node node, Node target, str key=None): +cpdef list node_find_target(MappingNode node, Node target, str key=None): if key is not None: target = target.value[key] @@ -1317,7 +1428,7 @@ cpdef list node_find_target(Node node, Node target, str key=None): # Helper for node_find_target() which walks a value -cdef bint _walk_find_target(Node node, list path, Node target): +cdef bint _walk_find_target(Node node, list path, Node target) except *: if node.file_index == target.file_index and node.line == target.line and node.column == target.column: return True elif type(node.value) is dict: @@ -1341,7 +1452,7 @@ cdef bint _walk_list_node(Node node, list path, Node target): # Helper for node_find_target() which walks a mapping -cdef bint _walk_dict_node(Node node, list path, Node target): +cdef bint _walk_dict_node(MappingNode node, list path, Node target): cdef str k cdef Node v diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py index 158f5fc11..48be1b83a 100644 --- a/src/buildstream/buildelement.py +++ b/src/buildstream/buildelement.py @@ -281,14 +281,11 @@ class BuildElement(Element): # Private Local Methods # ############################################################# def __get_commands(self, node, name): - list_node = self.node_get_member(node, list, name, []) - commands = [] - - for i in range(len(list_node)): - command = self.node_subst_list_element(node, name, [i]) - commands.append(command) - - return commands + raw_commands = node.get_sequence(name, []).as_str_list() + return [ + self.substitute_variables(command) + for command in raw_commands + ] def __run_command(self, sandbox, cmd): # Note the -e switch to 'sh' means to exit with an error diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 60aefecad..3029c98cd 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -485,6 +485,9 @@ class Element(Plugin): return None + def substitute_variables(self, value): + return self.__variables.subst(value) + def node_subst_member(self, node, member_name, default=_yaml._sentinel): """Fetch the value of a string node member, substituting any variables in the loaded value with the element contextual variables. @@ -500,10 +503,6 @@ class Element(Plugin): Raises: :class:`.LoadError`: When *member_name* is not found and no *default* was provided - This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_member` - except that it assumes the expected type is a string and will also perform variable - substitutions. - **Example:** .. code:: python @@ -512,7 +511,7 @@ class Element(Plugin): # variables in the returned string name = self.node_subst_member(node, 'name') """ - value = self.node_get_member(node, str, member_name, default) + value = node.get_str(member_name, default) try: return self.__variables.subst(value) except LoadError as e: @@ -532,61 +531,16 @@ class Element(Plugin): Raises: :class:`.LoadError` - This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_member` - except that it assumes the expected type is a list of strings and will also - perform variable substitutions. """ - value = self.node_get_member(node, list, member_name) ret = [] - for index, x in enumerate(value): + for value in node.get_sequence(member_name): try: - ret.append(self.__variables.subst(x)) + ret.append(self.__variables.subst(value.as_str())) except LoadError as e: - provenance = _yaml.node_get_provenance(node, key=member_name, indices=[index]) + provenance = _yaml.node_get_provenance(value) raise LoadError(e.reason, '{}: {}'.format(provenance, e), detail=e.detail) from e return ret - def node_subst_list_element(self, node, member_name, indices): - """Fetch the value of a list element from a node member, substituting any variables - in the loaded value with the element contextual variables. - - Args: - node (dict): A dictionary loaded from YAML - member_name (str): The name of the member to fetch - indices (list of int): List of indices to search, in case of nested lists - - Returns: - The value of the list element in *member_name* at the specified *indices* - - Raises: - :class:`.LoadError` - - This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_list_element` - except that it assumes the expected type is a string and will also perform variable - substitutions. - - **Example:** - - .. code:: python - - # Fetch the list itself - strings = self.node_get_member(node, list, 'strings') - - # Iterate over the list indices - for i in range(len(strings)): - - # Fetch the strings in this list, substituting content - # with our element's variables if needed - string = self.node_subst_list_element( - node, 'strings', [ i ]) - """ - value = self.node_get_list_element(node, str, member_name, indices) - try: - return self.__variables.subst(value) - except LoadError as e: - provenance = _yaml.node_get_provenance(node, key=member_name, indices=indices) - raise LoadError(e.reason, '{}: {}'.format(provenance, e), detail=e.detail) from e - def compute_manifest(self, *, include=None, exclude=None, orphans=True): """Compute and return this element's selective manifest @@ -854,9 +808,9 @@ class Element(Plugin): if bstdata is not None: with sandbox.batch(SandboxFlags.NONE): - commands = self.node_get_member(bstdata, list, 'integration-commands', []) - for i in range(len(commands)): - cmd = self.node_subst_list_element(bstdata, 'integration-commands', [i]) + commands = bstdata.get_sequence('integration-commands', []).as_str_list() + for command in commands: + cmd = self.substitute_variables(command) sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/', label=cmd) @@ -896,7 +850,7 @@ class Element(Plugin): if self.__dynamic_public is None: self.__load_public_data() - data = _yaml.node_get(self.__dynamic_public, dict, domain, default_value=None) + data = self.__dynamic_public.get_mapping(domain, default=None) if data is not None: data = _yaml.node_copy(data) @@ -2539,9 +2493,9 @@ class Element(Plugin): @classmethod def __compose_default_splits(cls, project, defaults, is_junction): - element_public = _yaml.node_get(defaults, dict, 'public', default_value={}) - element_bst = _yaml.node_get(element_public, dict, 'bst', default_value={}) - element_splits = _yaml.node_get(element_bst, dict, 'split-rules', default_value={}) + element_public = defaults.get_mapping("public", default={}) + element_bst = element_public.get_mapping("bst", default={}) + element_splits = element_bst.get_mapping("split-rules", default={}) if is_junction: splits = _yaml.node_copy(element_splits) @@ -2581,7 +2535,7 @@ class Element(Plugin): else: elements = project.element_overrides - overrides = _yaml.node_get(elements, dict, kind, default_value=None) + overrides = elements.get_mapping(kind, default=None) if overrides: _yaml.composite(defaults, overrides) @@ -2593,7 +2547,7 @@ class Element(Plugin): # @classmethod def __extract_environment(cls, project, meta): - default_env = _yaml.node_get(cls.__defaults, dict, 'environment', default_value={}) + default_env = cls.__defaults.get_mapping("environment", default={}) if meta.is_junction: environment = _yaml.new_empty_node() @@ -2624,7 +2578,7 @@ class Element(Plugin): else: project_nocache = project.base_env_nocache - default_nocache = _yaml.node_get(cls.__defaults, list, 'environment-nocache', default_value=[]) + default_nocache = cls.__defaults.get_sequence('environment-nocache', default=[]).as_str_list() element_nocache = meta.env_nocache # Accumulate values from the element default, the project and the element @@ -2639,8 +2593,7 @@ class Element(Plugin): # @classmethod def __extract_variables(cls, project, meta): - default_vars = _yaml.node_get(cls.__defaults, dict, 'variables', - default_value={}) + default_vars = cls.__defaults.get_mapping('variables', default={}) if meta.is_junction: variables = _yaml.node_copy(project.first_pass_config.base_variables) @@ -2667,7 +2620,7 @@ class Element(Plugin): def __extract_config(cls, meta): # The default config is already composited with the project overrides - config = _yaml.node_get(cls.__defaults, dict, 'config', default_value={}) + config = cls.__defaults.get_mapping('config', default={}) config = _yaml.node_copy(config) _yaml.composite(config, meta.config) @@ -2693,7 +2646,7 @@ class Element(Plugin): host_os = platform.get_host_os() # The default config is already composited with the project overrides - sandbox_defaults = _yaml.node_get(cls.__defaults, dict, 'sandbox', default_value={}) + sandbox_defaults = cls.__defaults.get_mapping('sandbox', default={}) sandbox_defaults = _yaml.node_copy(sandbox_defaults) _yaml.composite(sandbox_config, sandbox_defaults) @@ -2703,16 +2656,16 @@ class Element(Plugin): # Sandbox config, unlike others, has fixed members so we should validate them _yaml.node_validate(sandbox_config, ['build-uid', 'build-gid', 'build-os', 'build-arch']) - build_arch = _yaml.node_get(sandbox_config, str, 'build-arch', default_value=None) + build_arch = sandbox_config.get_str('build-arch', default=None) if build_arch: build_arch = Platform.canonicalize_arch(build_arch) else: build_arch = host_arch return SandboxConfig( - _yaml.node_get(sandbox_config, int, 'build-uid'), - _yaml.node_get(sandbox_config, int, 'build-gid'), - _yaml.node_get(sandbox_config, str, 'build-os', default_value=host_os), + sandbox_config.get_int('build-uid'), + sandbox_config.get_int('build-gid'), + sandbox_config.get_str('build-os', default=host_os), build_arch) # This makes a special exception for the split rules, which @@ -2720,15 +2673,15 @@ class Element(Plugin): # @classmethod def __extract_public(cls, meta): - base_public = _yaml.node_get(cls.__defaults, dict, 'public', default_value={}) + base_public = cls.__defaults.get_mapping('public', default={}) base_public = _yaml.node_copy(base_public) - base_bst = _yaml.node_get(base_public, dict, 'bst', default_value={}) - base_splits = _yaml.node_get(base_bst, dict, 'split-rules', default_value={}) + base_bst = base_public.get_mapping('bst', default={}) + base_splits = base_bst.get_mapping('split-rules', default={}) element_public = _yaml.node_copy(meta.public) - element_bst = _yaml.node_get(element_public, dict, 'bst', default_value={}) - element_splits = _yaml.node_get(element_bst, dict, 'split-rules', default_value={}) + element_bst = element_public.get_mapping('bst', default={}) + element_splits = element_bst.get_mapping('split-rules', default={}) # Allow elements to extend the default splits defined in their project or # element specific defaults @@ -2743,8 +2696,8 @@ class Element(Plugin): # Expand the splits in the public data using the Variables in the element def __expand_splits(self, element_public): - element_bst = _yaml.node_get(element_public, dict, 'bst', default_value={}) - element_splits = _yaml.node_get(element_bst, dict, 'split-rules', default_value={}) + element_bst = element_public.get_mapping('bst', default={}) + element_splits = element_bst.get_mapping('split-rules', default={}) # Resolve any variables in the public split rules directly for domain, splits in self.node_items(element_splits): @@ -2758,7 +2711,7 @@ class Element(Plugin): def __init_splits(self): bstdata = self.get_public_data('bst') - splits = self.node_get_member(bstdata, dict, 'split-rules') + splits = bstdata.get_mapping('split-rules') self.__splits = { domain: re.compile('^(?:' + '|'.join([utils._glob2re(r) for r in rules]) + ')$') for domain, rules in self.node_items(splits) @@ -2862,7 +2815,7 @@ class Element(Plugin): # If this ever changes, things will go wrong unexpectedly. if not self.__whitelist_regex: bstdata = self.get_public_data('bst') - whitelist = _yaml.node_get(bstdata, list, 'overlap-whitelist', default_value=[]) + whitelist = bstdata.get_sequence('overlap-whitelist', default=[]).as_str_list() whitelist_expressions = [utils._glob2re(self.__variables.subst(exp.strip())) for exp in whitelist] expression = ('^(?:' + '|'.join(whitelist_expressions) + ')$') self.__whitelist_regex = re.compile(expression) diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py index a12ff61ec..f2bd90b35 100644 --- a/src/buildstream/plugin.py +++ b/src/buildstream/plugin.py @@ -280,13 +280,7 @@ class Plugin(): Plugin implementors should implement this method to read configuration data and store it. - Plugins should use the :func:`Plugin.node_get_member() <buildstream.plugin.Plugin.node_get_member>` - and :func:`Plugin.node_get_list_element() <buildstream.plugin.Plugin.node_get_list_element>` - methods to fetch values from the passed `node`. This will ensure that a nice human readable error - message will be raised if the expected configuration is not found, indicating the filename, - line and column numbers. - - Further the :func:`Plugin.node_validate() <buildstream.plugin.Plugin.node_validate>` method + The :func:`Plugin.node_validate() <buildstream.plugin.Plugin.node_validate>` method should be used to ensure that the user has not specified keys in `node` which are unsupported by the plugin. @@ -294,8 +288,7 @@ class Plugin(): For Elements, when variable substitution is desirable, the :func:`Element.node_subst_member() <buildstream.element.Element.node_subst_member>` - and :func:`Element.node_subst_list_element() <buildstream.element.Element.node_subst_list_element>` - methods can be used. + method can be used. """ raise ImplError("{tag} plugin '{kind}' does not implement configure()".format( tag=self.__type_tag, kind=self.get_kind())) @@ -387,38 +380,6 @@ class Plugin(): provenance = _yaml.node_get_provenance(node, key=member_name) return str(provenance) - def node_get_member(self, node, expected_type, member_name, default=_yaml._sentinel, *, allow_none=False): - """Fetch the value of a node member, raising an error if the value is - missing or incorrectly typed. - - Args: - node (Node): A dictionary loaded from YAML - expected_type (type): The expected type of the node member - member_name (str): The name of the member to fetch - default (expected_type): A value to return when *member_name* is not specified in *node* - allow_none (bool): Allow explicitly set None values in the YAML (*Since: 1.4*) - - Returns: - The value of *member_name* in *node*, otherwise *default* - - Raises: - :class:`.LoadError`: When *member_name* is not found and no *default* was provided - - Note: - Returned strings are stripped of leading and trailing whitespace - - **Example:** - - .. code:: python - - # Expect a string 'name' in 'node' - name = self.node_get_member(node, str, 'name') - - # Fetch an optional integer - level = self.node_get_member(node, int, 'level', -1) - """ - return _yaml.node_get(node, expected_type, member_name, default_value=default, allow_none=allow_none) - def node_set_member(self, node, key, value): """Set the value of a node member Args: @@ -530,41 +491,6 @@ class Plugin(): """ _yaml.node_validate(node, valid_keys) - def node_get_list_element(self, node, expected_type, member_name, indices): - """Fetch the value of a list element from a node member, raising an error if the - value is incorrectly typed. - - Args: - node (dict): A dictionary loaded from YAML - expected_type (type): The expected type of the node member - member_name (str): The name of the member to fetch - indices (list of int): List of indices to search, in case of nested lists - - Returns: - The value of the list element in *member_name* at the specified *indices* - - Raises: - :class:`.LoadError` - - Note: - Returned strings are stripped of leading and trailing whitespace - - **Example:** - - .. code:: python - - # Fetch the list itself - things = self.node_get_member(node, list, 'things') - - # Iterate over the list indices - for i in range(len(things)): - - # Fetch dict things - thing = self.node_get_list_element( - node, dict, 'things', [ i ]) - """ - return _yaml.node_get(node, expected_type, member_name, indices=indices) - def debug(self, brief, *, detail=None): """Print a debugging message @@ -894,10 +820,10 @@ class Plugin(): project = self.__project for key, value in self.node_items(project.element_overrides): - if _yaml.node_get(value, bool, 'suppress-deprecation-warnings', default_value=False): + if value.get_bool('suppress-deprecation-warnings', default=False): silenced_warnings.add(key) for key, value in self.node_items(project.source_overrides): - if _yaml.node_get(value, bool, 'suppress-deprecation-warnings', default_value=False): + if value.get_bool('suppress-deprecation-warnings', default=False): silenced_warnings.add(key) return self.get_kind() in silenced_warnings diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py index b672cde0c..a9f18875f 100644 --- a/src/buildstream/plugins/elements/compose.py +++ b/src/buildstream/plugins/elements/compose.py @@ -65,10 +65,10 @@ class ComposeElement(Element): # We name this variable 'integration' only to avoid # collision with the Element.integrate() method. - self.integration = self.node_get_member(node, bool, 'integrate') - self.include = self.node_get_member(node, list, 'include') - self.exclude = self.node_get_member(node, list, 'exclude') - self.include_orphans = self.node_get_member(node, bool, 'include-orphans') + self.integration = node.get_bool('integrate') + self.include = node.get_sequence('include').as_str_list() + self.exclude = node.get_sequence('exclude').as_str_list() + self.include_orphans = node.get_bool('include-orphans') def preflight(self): pass diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py index 45847e685..bba56f351 100644 --- a/src/buildstream/plugins/elements/filter.py +++ b/src/buildstream/plugins/elements/filter.py @@ -171,9 +171,9 @@ class FilterElement(Element): 'include', 'exclude', 'include-orphans' ]) - self.include = self.node_get_member(node, list, 'include') - self.exclude = self.node_get_member(node, list, 'exclude') - self.include_orphans = self.node_get_member(node, bool, 'include-orphans') + self.include = node.get_sequence('include').as_str_list() + self.exclude = node.get_sequence('exclude').as_str_list() + self.include_orphans = node.get_bool('include-orphans') self.include_provenance = self.node_provenance(node, member_name='include') self.exclude_provenance = self.node_provenance(node, member_name='exclude') @@ -217,7 +217,7 @@ class FilterElement(Element): for dep in self.dependencies(Scope.BUILD, recurse=False): # Check that all the included/excluded domains exist pub_data = dep.get_public_data('bst') - split_rules = self.node_get_member(pub_data, dict, 'split-rules', {}) + split_rules = pub_data.get_mapping('split-rules', {}) unfound_includes = [] for domain in self.include: if domain not in split_rules: diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index 4222de360..d30e34828 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -175,9 +175,9 @@ class JunctionElement(Element): BST_FORBID_RDEPENDS = True def configure(self, node): - self.path = self.node_get_member(node, str, 'path', default='') - self.options = self.node_get_member(node, dict, 'options', default={}) - self.target = self.node_get_member(node, str, 'target', default=None) + self.path = node.get_str('path', default='') + self.options = node.get_mapping('options', default={}) + self.target = node.get_str('target', default=None) self.target_element = None self.target_junction = None diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py index 0d194dcc1..1c694060c 100644 --- a/src/buildstream/plugins/elements/script.py +++ b/src/buildstream/plugins/elements/script.py @@ -46,7 +46,7 @@ class ScriptElement(buildstream.ScriptElement): BST_VIRTUAL_DIRECTORY = True def configure(self, node): - for n in self.node_get_member(node, list, 'layout', []): + for n in node.get_sequence('layout', []): dst = self.node_subst_member(n, 'destination') elm = self.node_subst_member(n, 'element', None) self.layout_add(elm, dst) @@ -60,8 +60,7 @@ class ScriptElement(buildstream.ScriptElement): self.set_work_dir() self.set_install_root() - self.set_root_read_only(self.node_get_member(node, bool, - 'root-read-only', False)) + self.set_root_read_only(node.get_bool('root-read-only', default=False)) # Plugin entry point diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py index b9b15e268..7b1785979 100644 --- a/src/buildstream/plugins/sources/_downloadablefilesource.py +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -77,8 +77,8 @@ class DownloadableFileSource(Source): __urlopener = None def configure(self, node): - self.original_url = self.node_get_member(node, str, 'url') - self.ref = self.node_get_member(node, str, 'ref', None) + self.original_url = node.get_str('url') + self.ref = node.get_str('ref', None) self.url = self.translate_url(self.original_url) self._warn_deprecated_etag(node) @@ -99,7 +99,7 @@ class DownloadableFileSource(Source): return Consistency.RESOLVED def load_ref(self, node): - self.ref = self.node_get_member(node, str, 'ref', None) + self.ref = node.get_str('ref', None) self._warn_deprecated_etag(node) def get_ref(self): @@ -143,7 +143,7 @@ class DownloadableFileSource(Source): .format(self.url, sha256, self.ref)) def _warn_deprecated_etag(self, node): - etag = self.node_get_member(node, str, 'etag', None) + etag = node.get_str('etag', None) if etag: provenance = self.node_provenance(node, member_name='etag') self.warn('{} "etag" is deprecated and ignored.'.format(provenance)) diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py index e59986da6..28bea1415 100644 --- a/src/buildstream/plugins/sources/bzr.py +++ b/src/buildstream/plugins/sources/bzr.py @@ -69,9 +69,9 @@ class BzrSource(Source): def configure(self, node): self.node_validate(node, ['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS]) - self.original_url = self.node_get_member(node, str, 'url') - self.tracking = self.node_get_member(node, str, 'track') - self.ref = self.node_get_member(node, str, 'ref', None) + self.original_url = node.get_str('url') + self.tracking = node.get_str('track') + self.ref = node.get_str('ref', None) self.url = self.translate_url(self.original_url) def preflight(self): @@ -93,7 +93,7 @@ class BzrSource(Source): return Consistency.RESOLVED def load_ref(self, node): - self.ref = self.node_get_member(node, str, 'ref', None) + self.ref = node.get_str('ref', None) def get_ref(self): return self.ref diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py index e45994951..cc88cf53c 100644 --- a/src/buildstream/plugins/sources/deb.py +++ b/src/buildstream/plugins/sources/deb.py @@ -61,7 +61,7 @@ class DebSource(TarSource): def configure(self, node): super().configure(node) - self.base_dir = self.node_get_member(node, str, 'base-dir', None) + self.base_dir = node.get_str('base-dir', None) def preflight(self): return diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py index e42868264..01117db78 100644 --- a/src/buildstream/plugins/sources/patch.py +++ b/src/buildstream/plugins/sources/patch.py @@ -57,7 +57,7 @@ class PatchSource(Source): def configure(self, node): self.path = self.node_get_project_path(node, 'path', check_is_file=True) - self.strip_level = self.node_get_member(node, int, "strip-level", 1) + self.strip_level = node.get_int("strip-level", default=1) self.fullpath = os.path.join(self.get_project_directory(), self.path) def preflight(self): diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py index 9d6c40d74..627f26397 100644 --- a/src/buildstream/plugins/sources/pip.py +++ b/src/buildstream/plugins/sources/pip.py @@ -111,17 +111,17 @@ class PipSource(Source): def configure(self, node): self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] + Source.COMMON_CONFIG_KEYS) - self.ref = self.node_get_member(node, str, 'ref', None) - self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL) + self.ref = node.get_str('ref', None) + self.original_url = node.get_str('url', _PYPI_INDEX_URL) self.index_url = self.translate_url(self.original_url) - self.packages = self.node_get_member(node, list, 'packages', []) - self.requirements_files = self.node_get_member(node, list, 'requirements-files', []) + self.packages = node.get_sequence('packages', []).as_str_list() + self.requirements_files = node.get_sequence('requirements-files', []).as_str_list() if not (self.packages or self.requirements_files): raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self)) def preflight(self): - # Try to find a pip version that supports download command + # Try to find a pip version that spports download command self.host_pip = None for python in reversed(_PYTHON_VERSIONS): try: @@ -150,7 +150,7 @@ class PipSource(Source): return self.ref def load_ref(self, node): - self.ref = self.node_get_member(node, str, 'ref', None) + self.ref = node.get_str('ref', None) def set_ref(self, ref, node): node['ref'] = self.ref = ref diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py index 562a8f226..bb4c11325 100644 --- a/src/buildstream/plugins/sources/remote.py +++ b/src/buildstream/plugins/sources/remote.py @@ -62,8 +62,8 @@ class RemoteSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url)) - self.executable = self.node_get_member(node, bool, 'executable', False) + self.filename = node.get_str('filename', os.path.basename(self.url)) + self.executable = node.get_bool('executable', default=False) if os.sep in self.filename: raise SourceError('{}: filename parameter cannot contain directories'.format(self), diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py index c90de74ea..ef2ee1d85 100644 --- a/src/buildstream/plugins/sources/tar.py +++ b/src/buildstream/plugins/sources/tar.py @@ -72,8 +72,7 @@ class TarSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.base_dir = self.node_get_member(node, str, 'base-dir', '*') - + self.base_dir = node.get_str('base-dir', '*') self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) def preflight(self): diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py index 9981f1260..1724f0587 100644 --- a/src/buildstream/plugins/sources/zip.py +++ b/src/buildstream/plugins/sources/zip.py @@ -72,8 +72,7 @@ class ZipSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.base_dir = self.node_get_member(node, str, 'base-dir', '*') - + self.base_dir = node.get_str('base-dir', '*') self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) def get_unique_key(self): diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index d90b164bc..37ba8506b 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -112,7 +112,7 @@ class SandboxRemote(Sandbox): def specs_from_config_node(config_node, basedir=None): def require_node(config, keyname): - val = _yaml.node_get(config, dict, keyname, default_value=None) + val = config.get_mapping(keyname, default=None) if val is None: provenance = _yaml.node_get_provenance(remote_config, key=keyname) raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA, @@ -121,7 +121,7 @@ class SandboxRemote(Sandbox): .format(str(provenance), keyname)) return val - remote_config = _yaml.node_get(config_node, dict, 'remote-execution', default_value=None) + remote_config = config_node.get_mapping('remote-execution', default=None) if remote_config is None: return None @@ -131,7 +131,7 @@ class SandboxRemote(Sandbox): exec_config = require_node(remote_config, 'execution-service') storage_config = require_node(remote_config, 'storage-service') - action_config = _yaml.node_get(remote_config, dict, 'action-cache-service', default_value={}) + action_config = remote_config.get_mapping('action-cache-service', default={}) tls_keys = ['client-key', 'client-cert', 'server-cert'] @@ -173,7 +173,7 @@ class SandboxRemote(Sandbox): for tls_key in tls_keys: if tls_key in config: - _yaml.node_set(config, tls_key, resolve_path(_yaml.node_get(config, str, tls_key))) + _yaml.node_set(config, tls_key, resolve_path(config.get_str(tls_key))) return RemoteExecutionSpec(*[_yaml.node_sanitize(conf) for conf in service_configs]) diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 9fc9cf17d..609836551 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -258,8 +258,7 @@ class Source(Plugin): All Sources derive from this class, this interface defines how the core will be interacting with Sources. """ - __defaults = {} # The defaults from the project - __defaults_set = False # Flag, in case there are not defaults at all + __defaults = None # The defaults from the project BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False """Whether access to previous sources is required during track @@ -1274,20 +1273,19 @@ class Source(Plugin): @classmethod def __init_defaults(cls, project, meta): - if not cls.__defaults_set: + if cls.__defaults is None: if meta.first_pass: sources = project.first_pass_config.source_overrides else: sources = project.source_overrides - cls.__defaults = _yaml.node_get(sources, dict, meta.kind, default_value={}) - cls.__defaults_set = True + cls.__defaults = sources.get_mapping(meta.kind, default={}) # This will resolve the final configuration to be handed # off to source.configure() # @classmethod def __extract_config(cls, meta): - config = _yaml.node_get(cls.__defaults, dict, 'config', default_value={}) + config = cls.__defaults.get_mapping('config', default={}) config = _yaml.node_copy(config) _yaml.composite(config, meta.config) diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py index d682bb2ef..afbadbef7 100644 --- a/src/buildstream/testing/_sourcetests/mirror.py +++ b/src/buildstream/testing/_sourcetests/mirror.py @@ -357,9 +357,9 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): # Tracking tries upstream first. Check the ref is from upstream. new_element = _yaml.load(element_path) - source = _yaml.node_get(new_element, dict, 'sources', [0]) + source = new_element.get_sequence('sources').mapping_at(0) if 'ref' in source: - assert _yaml.node_get(source, str, 'ref') == upstream_ref + assert source.get_str('ref') == upstream_ref @pytest.mark.datafiles(DATA_DIR) @@ -422,6 +422,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): # Check that tracking fell back to the mirror new_element = _yaml.load(element_path) - source = _yaml.node_get(new_element, dict, 'sources', [0]) + source = new_element.get_sequence('sources').mapping_at(0) if 'ref' in source: - assert _yaml.node_get(source, str, 'ref') == mirror_ref + assert source.get_str('ref') == mirror_ref diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py index 668ea29e5..58679e9f1 100644 --- a/src/buildstream/testing/_sourcetests/track.py +++ b/src/buildstream/testing/_sourcetests/track.py @@ -321,13 +321,13 @@ def test_track_include(cli, tmpdir, datafiles, ref_storage, kind): # Get all of the sources assert 'sources' in new_sources - sources_list = _yaml.node_get(new_sources, list, 'sources') + sources_list = new_sources.get_sequence('sources') assert len(sources_list) == 1 # Get the first source from the sources list - new_source = _yaml.node_get(new_sources, dict, 'sources', indices=[0]) + new_source = sources_list.mapping_at(0) assert 'ref' in new_source - assert ref == _yaml.node_get(new_source, str, 'ref') + assert ref == new_source.get_str('ref') @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/elements/filter.py b/tests/elements/filter.py index d89c834e0..f9c80afff 100644 --- a/tests/elements/filter.py +++ b/tests/elements/filter.py @@ -239,8 +239,8 @@ def test_filter_track(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0]) - new_input_ref = _yaml.node_get(source_node, str, 'ref') + source_node = new_input.get_sequence('sources').mapping_at(0) + new_input_ref = source_node.get_str('ref') assert new_input_ref == ref @@ -294,7 +294,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0]) + source_node = new_input.get_sequence('sources').mapping_at(0) assert 'ref' not in source_node @@ -348,8 +348,8 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0]) - new_ref = _yaml.node_get(source_node, str, 'ref') + source_node = new_input.get_sequence('sources').mapping_at(0) + new_ref = source_node.get_str('ref') assert new_ref == ref @@ -413,13 +413,13 @@ def test_filter_track_multi(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0]) - new_ref = _yaml.node_get(source_node, str, 'ref') + source_node = new_input.get_sequence('sources').mapping_at(0) + new_ref = source_node.get_str('ref') assert new_ref == ref new_input2 = _yaml.load(input2_file) - source_node2 = _yaml.node_get(new_input2, dict, 'sources', indices=[0]) - new_ref2 = _yaml.node_get(source_node2, str, 'ref') + source_node2 = new_input2.get_sequence('sources').mapping_at(0) + new_ref2 = source_node2.get_str('ref') assert new_ref2 == ref @@ -482,12 +482,12 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0]) + source_node = new_input.get_sequence('sources').mapping_at(0) assert 'ref' not in source_node new_input2 = _yaml.load(input2_file) - source_node2 = _yaml.node_get(new_input2, dict, 'sources', indices=[0]) - new_ref2 = _yaml.node_get(source_node2, str, 'ref') + source_node2 = new_input2.get_sequence('sources').mapping_at(0) + new_ref2 = source_node2.get_str('ref') assert new_ref2 == ref diff --git a/tests/elements/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py index c6d128b72..b4a556244 100644 --- a/tests/elements/filter/basic/element_plugins/dynamic.py +++ b/tests/elements/filter/basic/element_plugins/dynamic.py @@ -5,7 +5,7 @@ from buildstream import Element, Scope class DynamicElement(Element): def configure(self, node): self.node_validate(node, ['split-rules']) - self.split_rules = self.node_get_member(node, dict, 'split-rules') + self.split_rules = node.get_mapping('split-rules') def preflight(self): pass diff --git a/tests/format/include.py b/tests/format/include.py index bfadce7ed..e2e16c34a 100644 --- a/tests/format/include.py +++ b/tests/format/include.py @@ -28,7 +28,7 @@ def test_include_project_file(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, bool, 'included') + assert loaded.get_bool('included') def test_include_missing_file(cli, tmpdir): @@ -87,7 +87,7 @@ def test_include_junction_file(cli, tmpdir, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, bool, 'included') + assert loaded.get_bool('included') @pytest.mark.datafiles(DATA_DIR) @@ -102,7 +102,7 @@ def test_include_junction_options(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'build_arch') == 'x86_64' + assert loaded.get_str('build_arch') == 'x86_64' @pytest.mark.datafiles(DATA_DIR) @@ -135,7 +135,7 @@ def test_junction_element_partial_project_project(cli, tmpdir, datafiles): 'junction.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'included', default_value=None) is None + assert loaded.get_str('included', default=None) is None @pytest.mark.datafiles(DATA_DIR) @@ -168,7 +168,7 @@ def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles): 'junction.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'included', default_value=None) is not None + assert loaded.get_str('included', default=None) is not None @pytest.mark.datafiles(DATA_DIR) @@ -182,8 +182,8 @@ def test_include_element_overrides(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'manual_main_override', default_value=None) is not None - assert _yaml.node_get(loaded, str, 'manual_included_override', default_value=None) is not None + assert loaded.get_str('manual_main_override', default=None) is not None + assert loaded.get_str('manual_included_override', default=None) is not None @pytest.mark.datafiles(DATA_DIR) @@ -197,7 +197,7 @@ def test_include_element_overrides_composition(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, list, 'build-commands') == ['first', 'second'] + assert loaded.get_sequence('build-commands').as_str_list() == ['first', 'second'] @pytest.mark.datafiles(DATA_DIR) @@ -213,9 +213,9 @@ def test_list_overide_does_not_fail_upon_first_composition(cli, datafiles): loaded = _yaml.load_data(result.output) # Assert that the explicitly overwritten public data is present - bst = _yaml.node_get(loaded, dict, 'bst') + bst = loaded.get_mapping('bst') assert 'foo-commands' in bst - assert _yaml.node_get(bst, list, 'foo-commands') == ['need', 'this'] + assert bst.get_sequence('foo-commands').as_str_list() == ['need', 'this'] @pytest.mark.datafiles(DATA_DIR) @@ -229,7 +229,7 @@ def test_include_element_overrides_sub_include(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'included', default_value=None) is not None + assert loaded.get_str('included', default=None) is not None @pytest.mark.datafiles(DATA_DIR) @@ -248,8 +248,8 @@ def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles): 'junction.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'main_override', default_value=None) is not None - assert _yaml.node_get(loaded, str, 'included_override', default_value=None) is None + assert loaded.get_str('main_override', default=None) is not None + assert loaded.get_str('included_override', default=None) is None @pytest.mark.datafiles(DATA_DIR) @@ -264,7 +264,7 @@ def test_conditional_in_fragment(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'size') == '8' + assert loaded.get_str('size') == '8' @pytest.mark.datafiles(DATA_DIR) @@ -278,7 +278,7 @@ def test_inner(cli, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'build_arch') == 'x86_64' + assert loaded.get_str('build_arch') == 'x86_64' @pytest.mark.datafiles(DATA_DIR) @@ -310,4 +310,4 @@ def test_local_to_junction(cli, tmpdir, datafiles): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, bool, 'included') + assert loaded.get_bool('included') diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py index 4afde817d..b922c7a0e 100644 --- a/tests/format/include_composition.py +++ b/tests/format/include_composition.py @@ -14,7 +14,7 @@ def make_includes(basedir): return Includes(loader) -def test_main_has_prority(tmpdir): +def test_main_has_priority(tmpdir): includes = make_includes(str(tmpdir)) _yaml.dump({'(@)': ['a.yml'], @@ -28,7 +28,7 @@ def test_main_has_prority(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['main'] + assert main.get_sequence('test').as_str_list() == ['main'] def test_include_cannot_append(tmpdir): @@ -44,7 +44,7 @@ def test_include_cannot_append(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['main'] + assert main.get_sequence('test').as_str_list() == ['main'] def test_main_can_append(tmpdir): @@ -60,7 +60,7 @@ def test_main_can_append(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['a', 'main'] + assert main.get_sequence('test').as_str_list() == ['a', 'main'] def test_sibling_cannot_append_backward(tmpdir): @@ -77,7 +77,7 @@ def test_sibling_cannot_append_backward(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['b'] + assert main.get_sequence('test').as_str_list() == ['b'] def test_sibling_can_append_forward(tmpdir): @@ -94,7 +94,7 @@ def test_sibling_can_append_forward(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['a', 'b'] + assert main.get_sequence('test').as_str_list() == ['a', 'b'] def test_lastest_sibling_has_priority(tmpdir): @@ -111,7 +111,7 @@ def test_lastest_sibling_has_priority(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['b'] + assert main.get_sequence('test').as_str_list() == ['b'] def test_main_keeps_keys(tmpdir): @@ -127,8 +127,8 @@ def test_main_keeps_keys(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['a'] - assert _yaml.node_get(main, str, 'something') == 'else' + assert main.get_sequence('test').as_str_list() == ['a'] + assert main.get_str('something') == 'else' def test_overwrite_directive_on_later_composite(tmpdir): @@ -151,5 +151,5 @@ def test_overwrite_directive_on_later_composite(tmpdir): includes.process(main) - assert _yaml.node_get(main, list, 'test') == ['Overwritten'] - assert _yaml.node_get(main, str, 'foo') == 'should be present' + assert main.get_sequence('test').as_str_list() == ['Overwritten'] + assert main.get_str('foo') == 'should be present' diff --git a/tests/format/optionarch.py b/tests/format/optionarch.py index a35ac685d..f347e27ae 100644 --- a/tests/format/optionarch.py +++ b/tests/format/optionarch.py @@ -48,7 +48,7 @@ def test_conditional(cli, datafiles, machine, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'result') == expected + assert loaded.get_str('result') == expected @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/format/optionbool.py b/tests/format/optionbool.py index 0d1ee601e..d772b483c 100644 --- a/tests/format/optionbool.py +++ b/tests/format/optionbool.py @@ -42,7 +42,7 @@ def test_conditional_cli(cli, datafiles, target, option, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'thepony') == expected + assert loaded.get_str('thepony') == expected # Test configuration of boolean option in the config file @@ -71,7 +71,7 @@ def test_conditional_config(cli, datafiles, target, option, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'thepony') == expected + assert loaded.get_str('thepony') == expected @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/format/optioneltmask.py b/tests/format/optioneltmask.py index d33b5771c..75265fdd7 100644 --- a/tests/format/optioneltmask.py +++ b/tests/format/optioneltmask.py @@ -28,7 +28,7 @@ def test_conditional_cli(cli, datafiles, target, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'debug') == expected + assert loaded.get_str('debug') == expected @pytest.mark.datafiles(DATA_DIR) @@ -56,7 +56,7 @@ def test_conditional_config(cli, datafiles, target, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'debug') == expected + assert loaded.get_str('debug') == expected @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/format/optionenum.py b/tests/format/optionenum.py index b8a96b0c2..f9aff503f 100644 --- a/tests/format/optionenum.py +++ b/tests/format/optionenum.py @@ -33,7 +33,7 @@ def test_conditional_cli(cli, datafiles, target, option, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'result') == expected + assert loaded.get_str('result') == expected @pytest.mark.datafiles(DATA_DIR) @@ -66,7 +66,7 @@ def test_conditional_config(cli, datafiles, target, option, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'result') == expected + assert loaded.get_str('result') == expected @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/format/optionexports.py b/tests/format/optionexports.py index 5df7522b5..104abcf83 100644 --- a/tests/format/optionexports.py +++ b/tests/format/optionexports.py @@ -36,4 +36,4 @@ def test_export(cli, datafiles, option_name, option_value, var_name, var_value): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, var_name) == var_value + assert loaded.get_str(var_name) == var_value diff --git a/tests/format/optionflags.py b/tests/format/optionflags.py index e28c54236..29bb7ec2c 100644 --- a/tests/format/optionflags.py +++ b/tests/format/optionflags.py @@ -39,7 +39,7 @@ def test_conditional_cli(cli, datafiles, target, option, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'result') == expected + assert loaded.get_str('result') == expected @pytest.mark.datafiles(DATA_DIR) @@ -69,7 +69,7 @@ def test_conditional_config(cli, datafiles, target, option, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'result') == expected + assert loaded.get_str('result') == expected @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/format/optionos.py b/tests/format/optionos.py index 57277b106..f915d889e 100644 --- a/tests/format/optionos.py +++ b/tests/format/optionos.py @@ -47,7 +47,7 @@ def test_conditionals(cli, datafiles, system, value, expected): result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'result') == expected + assert loaded.get_str('result') == expected @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/format/optionoverrides.py b/tests/format/optionoverrides.py index 60d02b3a3..d4ed257dd 100644 --- a/tests/format/optionoverrides.py +++ b/tests/format/optionoverrides.py @@ -29,4 +29,4 @@ def test_override(cli, datafiles, arch): expected_value = '--host={}-unknown-linux-gnu'.format(arch) loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'conf-global') == expected_value + assert loaded.get_str('conf-global') == expected_value diff --git a/tests/format/options.py b/tests/format/options.py index 3a8210dc3..9c0e043f9 100644 --- a/tests/format/options.py +++ b/tests/format/options.py @@ -136,7 +136,7 @@ def test_simple_conditional(cli, datafiles, opt_option, expected_prefix): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'prefix') == expected_prefix + assert loaded.get_str('prefix') == expected_prefix @pytest.mark.datafiles(DATA_DIR) @@ -159,7 +159,7 @@ def test_nested_conditional(cli, datafiles, debug, logging, expected): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'debug') == expected + assert loaded.get_str('debug') == expected @pytest.mark.datafiles(DATA_DIR) @@ -182,7 +182,7 @@ def test_compound_and_conditional(cli, datafiles, debug, logging, expected): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'debug') == expected + assert loaded.get_str('debug') == expected @pytest.mark.datafiles(DATA_DIR) @@ -205,7 +205,7 @@ def test_compound_or_conditional(cli, datafiles, debug, logging, expected): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - assert _yaml.node_get(loaded, str, 'logging') == expected + assert loaded.get_str('logging') == expected @pytest.mark.datafiles(DATA_DIR) @@ -223,10 +223,10 @@ def test_deep_nesting_level1(cli, datafiles, option, expected): 'element.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - shallow_list = _yaml.node_get(loaded, list, 'shallow-nest') - first_dict = shallow_list[0] + shallow_list = loaded.get_sequence('shallow-nest') + first_dict = shallow_list.mapping_at(0) - assert _yaml.node_get(first_dict, str, 'animal') == expected + assert first_dict.get_str('animal') == expected @pytest.mark.datafiles(DATA_DIR) @@ -244,8 +244,8 @@ def test_deep_nesting_level2(cli, datafiles, option, expected): 'element-deeper.bst']) result.assert_success() loaded = _yaml.load_data(result.output) - shallow_list = _yaml.node_get(loaded, list, 'deep-nest') - deeper_list = shallow_list[0] - first_dict = deeper_list[0] + shallow_list = loaded.get_sequence('deep-nest') + deeper_list = shallow_list.sequence_at(0) + first_dict = deeper_list.mapping_at(0) - assert _yaml.node_get(first_dict, str, 'animal') == expected + assert first_dict.get_str('animal') == expected diff --git a/tests/format/project.py b/tests/format/project.py index fbb742d47..697fe7e3d 100644 --- a/tests/format/project.py +++ b/tests/format/project.py @@ -82,8 +82,8 @@ def test_load_default_project(cli, datafiles): # Read back some of our project defaults from the env env = _yaml.load_data(result.output) - assert _yaml.node_get(env, str, 'USER') == "tomjon" - assert _yaml.node_get(env, str, 'TERM') == "dumb" + assert env.get_str('USER') == "tomjon" + assert env.get_str('TERM') == "dumb" @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -97,8 +97,8 @@ def test_load_project_from_subdir(cli, datafiles): # Read back some of our project defaults from the env env = _yaml.load_data(result.output) - assert _yaml.node_get(env, str, 'USER') == "tomjon" - assert _yaml.node_get(env, str, 'TERM') == "dumb" + assert env.get_str('USER') == "tomjon" + assert env.get_str('TERM') == "dumb" @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -111,7 +111,7 @@ def test_override_project_path(cli, datafiles): # Read back the overridden path env = _yaml.load_data(result.output) - assert _yaml.node_get(env, str, 'PATH') == "/bin:/sbin" + assert env.get_str('PATH') == "/bin:/sbin" @pytest.mark.datafiles(os.path.join(DATA_DIR)) diff --git a/tests/format/projectoverrides.py b/tests/format/projectoverrides.py index 4b0c3f4d0..730e43b1e 100644 --- a/tests/format/projectoverrides.py +++ b/tests/format/projectoverrides.py @@ -24,6 +24,6 @@ def test_prepend_configure_commands(cli, datafiles): result.assert_success() loaded = _yaml.load_data(result.output) - config_commands = _yaml.node_get(loaded, list, 'configure-commands') + config_commands = loaded.get_sequence('configure-commands').as_str_list() assert len(config_commands) == 3 assert config_commands[0] == 'echo "Hello World!"' diff --git a/tests/format/variables.py b/tests/format/variables.py index 87e3b2903..ed5407b83 100644 --- a/tests/format/variables.py +++ b/tests/format/variables.py @@ -49,7 +49,7 @@ def test_defaults(cli, datafiles, target, varname, expected): ]) result.assert_success() result_vars = _yaml.load_data(result.output) - assert _yaml.node_get(result_vars, str, varname) == expected + assert result_vars.get_str(varname) == expected ################################################################ @@ -75,7 +75,7 @@ def test_overrides(cli, datafiles, target, varname, expected): ]) result.assert_success() result_vars = _yaml.load_data(result.output) - assert _yaml.node_get(result_vars, str, varname) == expected + assert result_vars.get_str(varname) == expected @pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing_variables')) diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 97bce91a7..f9cd1cba4 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -15,7 +15,7 @@ from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream import utils -from tests.testutils import generate_junction, yaml_file_get_provenance, create_artifact_share +from tests.testutils import generate_junction, create_artifact_share from . import configure_project @@ -460,8 +460,9 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - provenance = yaml_file_get_provenance( - element_path, 'junction-dep.bst', key='depends', indices=[0]) + element_node = _yaml.load(element_path, shortname='junction-dep.bst') + ref_node = element_node.get_sequence('depends').mapping_at(0) + provenance = _yaml.node_get_provenance(ref_node) assert str(provenance) in result.stderr diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py index 81fd43487..38aafa558 100644 --- a/tests/frontend/cross_junction_workspace.py +++ b/tests/frontend/cross_junction_workspace.py @@ -75,11 +75,12 @@ def test_list_cross_junction(cli, tmpdir): result.assert_success() loaded = _yaml.load_data(result.output) - assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list) - workspaces = _yaml.node_get(loaded, list, 'workspaces') + workspaces = loaded.get_sequence('workspaces') assert len(workspaces) == 1 - assert 'element' in workspaces[0] - assert _yaml.node_get(workspaces[0], str, 'element') == element + first_workspace = workspaces.mapping_at(0) + + assert 'element' in first_workspace + assert first_workspace.get_str('element') == element def test_close_cross_junction(cli, tmpdir): @@ -97,8 +98,7 @@ def test_close_cross_junction(cli, tmpdir): result.assert_success() loaded = _yaml.load_data(result.output) - assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list) - workspaces = _yaml.node_get(loaded, list, 'workspaces') + workspaces = loaded.get_sequence('workspaces') assert not workspaces @@ -116,8 +116,7 @@ def test_close_all_cross_junction(cli, tmpdir): result.assert_success() loaded = _yaml.load_data(result.output) - assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list) - workspaces = _yaml.node_get(loaded, list, 'workspaces') + workspaces = loaded.get_sequence('workspaces') assert not workspaces diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py index f470ce2b7..d7e0ecafb 100644 --- a/tests/frontend/fetch.py +++ b/tests/frontend/fetch.py @@ -9,7 +9,7 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason -from tests.testutils import generate_junction, yaml_file_get_provenance +from tests.testutils import generate_junction from . import configure_project @@ -171,6 +171,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - provenance = yaml_file_get_provenance( - element_path, 'junction-dep.bst', key='depends', indices=[0]) + element_node = _yaml.load(element_path, shortname='junction-dep.bst') + ref_node = element_node.get_sequence('depends').mapping_at(0) + provenance = _yaml.node_get_provenance(ref_node) assert str(provenance) in result.stderr diff --git a/tests/frontend/init.py b/tests/frontend/init.py index 0eac5d528..e9d92be90 100644 --- a/tests/frontend/init.py +++ b/tests/frontend/init.py @@ -19,9 +19,9 @@ def test_defaults(cli, tmpdir): result.assert_success() project_conf = _yaml.load(project_path) - assert _yaml.node_get(project_conf, str, 'name') == 'foo' - assert _yaml.node_get(project_conf, str, 'format-version') == str(BST_FORMAT_VERSION) - assert _yaml.node_get(project_conf, str, 'element-path') == 'elements' + assert project_conf.get_str('name') == 'foo' + assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION) + assert project_conf.get_str('element-path') == 'elements' def test_all_options(cli, tmpdir): @@ -37,9 +37,9 @@ def test_all_options(cli, tmpdir): result.assert_success() project_conf = _yaml.load(project_path) - assert _yaml.node_get(project_conf, str, 'name') == 'foo' - assert _yaml.node_get(project_conf, str, 'format-version') == str(2) - assert _yaml.node_get(project_conf, str, 'element-path') == 'ponies' + assert project_conf.get_str('name') == 'foo' + assert project_conf.get_str('format-version') == str(2) + assert project_conf.get_str('element-path') == 'ponies' elements_dir = os.path.join(project, 'ponies') assert os.path.isdir(elements_dir) @@ -70,8 +70,8 @@ def test_force_overwrite_project(cli, tmpdir): result.assert_success() project_conf = _yaml.load(project_path) - assert _yaml.node_get(project_conf, str, 'name') == 'foo' - assert _yaml.node_get(project_conf, str, 'format-version') == str(BST_FORMAT_VERSION) + assert project_conf.get_str('name') == 'foo' + assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION) @pytest.mark.parametrize("project_name", [('Micheal Jackson'), ('one+one')]) @@ -122,6 +122,6 @@ def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path): assert full_element_path.exists() project_conf = _yaml.load(str(project_conf_path)) - assert _yaml.node_get(project_conf, str, 'name') == 'project_name' - assert _yaml.node_get(project_conf, str, 'format-version') == '0' - assert _yaml.node_get(project_conf, str, 'element-path') == element_path + assert project_conf.get_str('name') == 'project_name' + assert project_conf.get_str('format-version') == '0' + assert project_conf.get_str('element-path') == element_path diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py index f17c14029..7c0df4595 100644 --- a/tests/frontend/project/sources/fetch_source.py +++ b/tests/frontend/project/sources/fetch_source.py @@ -38,11 +38,11 @@ class FetchFetcher(SourceFetcher): class FetchSource(Source): # Read config to know which URLs to fetch def configure(self, node): - self.original_urls = self.node_get_member(node, list, 'urls') - self.output_file = self.node_get_member(node, str, 'output-text') + self.original_urls = node.get_sequence('urls').as_str_list() + self.output_file = node.get_str('output-text') self.fetch_succeeds = {} if 'fetch-succeeds' in node: - fetch_succeeds_node = self.node_get_member(node, dict, 'fetch-succeeds') + fetch_succeeds_node = node.get_mapping('fetch-succeeds') for key, value in self.node_items(fetch_succeeds_node): self.fetch_succeeds[key] = value in ('True', 'true') diff --git a/tests/frontend/show.py b/tests/frontend/show.py index 4ef97dd84..4192ba1e5 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -10,7 +10,7 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason -from tests.testutils import generate_junction, yaml_file_get_provenance +from tests.testutils import generate_junction from . import configure_project @@ -333,8 +333,9 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): etc_result.assert_success() else: # Assert that we have the expected provenance encoded into the error - provenance = yaml_file_get_provenance( - element_path, 'junction-dep.bst', key='depends', indices=[0]) + element_node = _yaml.load(element_path, shortname='junction-dep.bst') + ref_node = element_node.get_sequence('depends').mapping_at(0) + provenance = _yaml.node_get_provenance(ref_node) assert str(provenance) in dep_result.stderr dep_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) diff --git a/tests/frontend/track.py b/tests/frontend/track.py index 808bf0593..1163cc687 100644 --- a/tests/frontend/track.py +++ b/tests/frontend/track.py @@ -9,7 +9,7 @@ from buildstream.testing import create_repo from buildstream.testing import cli # pylint: disable=unused-import from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream import _yaml -from tests.testutils import generate_junction, yaml_file_get_provenance +from tests.testutils import generate_junction from . import configure_project # Project directory @@ -275,8 +275,9 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - provenance = yaml_file_get_provenance( - element_path, 'junction-dep.bst', key='depends', indices=[0]) + element_node = _yaml.load(element_path, shortname='junction-dep.bst') + ref_node = element_node.get_sequence('depends').mapping_at(0) + provenance = _yaml.node_get_provenance(ref_node) assert str(provenance) in result.stderr @@ -313,8 +314,9 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage): result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - provenance = yaml_file_get_provenance( - element_path, 'junction-dep.bst', key='depends', indices=[0]) + element_node = _yaml.load(element_path, shortname='junction-dep.bst') + ref_node = element_node.get_sequence('depends').mapping_at(0) + provenance = _yaml.node_get_provenance(ref_node) assert str(provenance) in result.stderr # Now track the junction itself diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py index 5267f8e56..b240e5728 100644 --- a/tests/frontend/workspace.py +++ b/tests/frontend/workspace.py @@ -184,11 +184,11 @@ def test_open_bzr_customize(cli, tmpdir, datafiles): # Check that the correct origin branch is set element_config = _yaml.load(os.path.join(project, "elements", element_name)) - source_config = _yaml.node_get(element_config, dict, 'sources', [0]) + source_config = element_config.get_sequence('sources').mapping_at(0) output = subprocess.check_output(["bzr", "info"], cwd=workspace) - stripped_url = _yaml.node_get(source_config, str, 'url').lstrip("file:///") + stripped_url = source_config.get_str('url').lstrip("file:///") expected_output_str = ("checkout of branch: /{}/{}" - .format(stripped_url, _yaml.node_get(source_config, str, 'track'))) + .format(stripped_url, source_config.get_str('track'))) assert expected_output_str in str(output) @@ -608,13 +608,12 @@ def test_list(cli, tmpdir, datafiles): result.assert_success() loaded = _yaml.load_data(result.output) - assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list) - workspaces = _yaml.node_get(loaded, list, 'workspaces') + workspaces = loaded.get_sequence('workspaces') assert len(workspaces) == 1 - space = workspaces[0] - assert _yaml.node_get(space, str, 'element') == element_name - assert _yaml.node_get(space, str, 'directory') == workspace + space = workspaces.mapping_at(0) + assert space.get_str('element') == element_name + assert space.get_str('directory') == workspace @pytest.mark.datafiles(DATA_DIR) @@ -1133,7 +1132,7 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element): # Delete the ref from the source so that we can detect if the # element has been tracked element_contents = _yaml.load(element_file) - _yaml.node_del(_yaml.node_get(element_contents, dict, 'sources', [0]), 'ref') + _yaml.node_del(element_contents.get_sequence('sources').mapping_at(0), 'ref') _yaml.dump(element_contents, element_file) result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm]) @@ -1141,7 +1140,7 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element): # Element is tracked now element_contents = _yaml.load(element_file) - assert 'ref' in _yaml.node_get(element_contents, dict, 'sources', [0]) + assert 'ref' in element_contents.get_sequence('sources').mapping_at(0) @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/internals/yaml.py b/tests/internals/yaml.py index cd3bdc535..aa83d949a 100644 --- a/tests/internals/yaml.py +++ b/tests/internals/yaml.py @@ -103,16 +103,16 @@ def test_node_get(datafiles): base = _yaml.load(filename) assert base.value.get('kind').value == 'pony' - children = _yaml.node_get(base, list, 'children') - assert isinstance(children, list) + children = base.get_sequence('children') + assert isinstance(children, _yaml.SequenceNode) assert len(children) == 7 - child = _yaml.node_get(base, dict, 'children', indices=[6]) + child = base.get_sequence('children').mapping_at(6) assert_provenance(filename, 20, 8, child, 'mood') - extra = _yaml.node_get(base, dict, 'extra') + extra = base.get_mapping('extra') with pytest.raises(LoadError) as exc: - _yaml.node_get(extra, dict, 'old') + extra.get_mapping('old') assert exc.value.reason == LoadErrorReason.INVALID_DATA @@ -128,7 +128,7 @@ def test_node_set(datafiles): assert 'mother' not in base _yaml.node_set(base, 'mother', 'snow white') - assert _yaml.node_get(base, str, 'mother') == 'snow white' + assert base.get_str('mother') == 'snow white' @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -141,14 +141,14 @@ def test_node_set_overwrite(datafiles): base = _yaml.load(filename) # Overwrite a string - assert _yaml.node_get(base, str, 'kind') == 'pony' + assert base.get_str('kind') == 'pony' _yaml.node_set(base, 'kind', 'cow') - assert _yaml.node_get(base, str, 'kind') == 'cow' + assert base.get_str('kind') == 'cow' # Overwrite a list as a string - assert _yaml.node_get(base, list, 'moods') == ['happy', 'sad'] + assert base.get_sequence('moods').as_str_list() == ['happy', 'sad'] _yaml.node_set(base, 'moods', 'unemotional') - assert _yaml.node_get(base, str, 'moods') == 'unemotional' + assert base.get_str('moods') == 'unemotional' @pytest.mark.datafiles(os.path.join(DATA_DIR)) @@ -160,13 +160,11 @@ def test_node_set_list_element(datafiles): base = _yaml.load(filename) - assert _yaml.node_get(base, list, 'moods') == ['happy', 'sad'] - assert _yaml.node_get(base, str, 'moods', indices=[0]) == 'happy' + assert base.get_sequence('moods').as_str_list() == ['happy', 'sad'] _yaml.node_set(base, 'moods', 'confused', indices=[0]) - assert _yaml.node_get(base, list, 'moods') == ['confused', 'sad'] - assert _yaml.node_get(base, str, 'moods', indices=[0]) == 'confused' + assert base.get_sequence('moods').as_str_list() == ['confused', 'sad'] # Really this is testing _yaml.node_copy(), we want to @@ -188,14 +186,14 @@ def test_composite_preserve_originals(datafiles): base_copy = _yaml.node_copy(base) _yaml.composite_dict(base_copy, overlay) - copy_extra = _yaml.node_get(base_copy, dict, 'extra') - orig_extra = _yaml.node_get(base, dict, 'extra') + copy_extra = base_copy.get_mapping('extra') + orig_extra = base.get_mapping('extra') # Test that the node copy has the overridden value... - assert _yaml.node_get(copy_extra, str, 'old') == 'override' + assert copy_extra.get_str('old') == 'override' # But the original node is not effected by the override. - assert _yaml.node_get(orig_extra, str, 'old') == 'new' + assert orig_extra.get_str('old') == 'new' # Tests for list composition @@ -254,11 +252,11 @@ def test_list_composition(datafiles, filename, tmpdir, _yaml.composite_dict(base, overlay) - children = _yaml.node_get(base, list, 'children') + children = base.get_sequence('children') assert len(children) == length - child = children[index] + child = children.mapping_at(index) - assert _yaml.node_get(child, str, 'mood') == mood + assert child.get_str('mood') == mood assert_provenance(prov_file, prov_line, prov_col, child, 'mood') @@ -272,7 +270,7 @@ def test_list_deletion(datafiles): overlay = _yaml.load(overlay, shortname='listoverwriteempty.yaml') _yaml.composite_dict(base, overlay) - children = _yaml.node_get(base, list, 'children') + children = base.get_sequence('children') assert not children @@ -286,8 +284,8 @@ def test_nonexistent_list_extension(datafiles): _yaml.node_extend_list(base, 'todo', 3, 'empty') - assert len(_yaml.node_get(base, list, 'todo')) == 3 - assert _yaml.node_get(base, list, 'todo') == ['empty', 'empty', 'empty'] + assert len(base.get_sequence('todo')) == 3 + assert base.get_sequence('todo').as_str_list() == ['empty', 'empty', 'empty'] # Tests for deep list composition @@ -390,11 +388,11 @@ def test_list_composition_twice(datafiles, tmpdir, filename1, filename2, _yaml.composite_dict(base, overlay1) _yaml.composite_dict(base, overlay2) - children = _yaml.node_get(base, list, 'children') + children = base.get_sequence('children') assert len(children) == length - child = children[index] + child = children.mapping_at(index) - assert _yaml.node_get(child, str, 'mood') == mood + assert child.get_str('mood') == mood assert_provenance(prov_file, prov_line, prov_col, child, 'mood') ##################### @@ -407,11 +405,11 @@ def test_list_composition_twice(datafiles, tmpdir, filename1, filename2, _yaml.composite_dict(overlay1, overlay2) _yaml.composite_dict(base, overlay1) - children = _yaml.node_get(base, list, 'children') + children = base.get_sequence('children') assert len(children) == length - child = children[index] + child = children.mapping_at(index) - assert _yaml.node_get(child, str, 'mood') == mood + assert child.get_str('mood') == mood assert_provenance(prov_file, prov_line, prov_col, child, 'mood') @@ -424,19 +422,19 @@ def test_convert_value_to_string(datafiles): # Run file through yaml to convert it test_dict = _yaml.load(conf_file) - user_config = _yaml.node_get(test_dict, str, "Test1") + user_config = test_dict.get_str("Test1") assert isinstance(user_config, str) assert user_config == "1_23_4" - user_config = _yaml.node_get(test_dict, str, "Test2") + user_config = test_dict.get_str("Test2") assert isinstance(user_config, str) assert user_config == "1.23.4" - user_config = _yaml.node_get(test_dict, str, "Test3") + user_config = test_dict.get_str("Test3") assert isinstance(user_config, str) assert user_config == "1.20" - user_config = _yaml.node_get(test_dict, str, "Test4") + user_config = test_dict.get_str("Test4") assert isinstance(user_config, str) assert user_config == "OneTwoThree" @@ -451,7 +449,7 @@ def test_value_doesnt_match_expected(datafiles): test_dict = _yaml.load(conf_file) with pytest.raises(LoadError) as exc: - _yaml.node_get(test_dict, int, "Test4") + test_dict.get_int("Test4") assert exc.value.reason == LoadErrorReason.INVALID_DATA diff --git a/tests/sources/git.py b/tests/sources/git.py index 45a2b827a..69155e02b 100644 --- a/tests/sources/git.py +++ b/tests/sources/git.py @@ -760,7 +760,7 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit): result.assert_success() element = _yaml.load(element_path) - new_ref = _yaml.node_get(_yaml.node_get(element, dict, 'sources', [0]), str, 'ref') + new_ref = element.get_sequence('sources').mapping_at(0).get_str('ref') if ref_format == 'git-describe' and tag: # Check and strip prefix @@ -849,16 +849,16 @@ def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type): if ref_storage == 'inline': element = _yaml.load(element_path) - tags = _yaml.node_get(_yaml.node_get(element, dict, 'sources', [0]), list, 'tags') + tags = element.get_sequence('sources').mapping_at(0).get_sequence('tags') assert len(tags) == 2 for tag in tags: assert 'tag' in tag assert 'commit' in tag assert 'annotated' in tag - assert _yaml.node_get(tag, bool, 'annotated') == (tag_type == 'annotated') + assert tag.get_bool('annotated') == (tag_type == 'annotated') - assert {(_yaml.node_get(tag, str, 'tag'), - _yaml.node_get(tag, str, 'commit')) + assert {(tag.get_str('tag'), + tag.get_str('commit')) for tag in tags} == {('tag1', repo.rev_parse('tag1^{commit}')), ('tag2', repo.rev_parse('tag2^{commit}'))} @@ -962,18 +962,18 @@ def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_ty if ref_storage == 'inline': element = _yaml.load(element_path) - source = _yaml.node_get(element, dict, 'sources', indices=[0]) - tags = _yaml.node_get(source, list, 'tags') + source = element.get_sequence('sources').mapping_at(0) + tags = source.get_sequence('tags') assert len(tags) == 1 - tag = _yaml.node_get(source, dict, 'tags', indices=[0]) + tag = source.get_sequence('tags').mapping_at(0) assert 'tag' in tag assert 'commit' in tag assert 'annotated' in tag - assert _yaml.node_get(tag, bool, 'annotated') == (tag_type == 'annotated') + assert tag.get_bool('annotated') == (tag_type == 'annotated') - tag_name = _yaml.node_get(tag, str, 'tag') - commit = _yaml.node_get(tag, str, 'commit') + tag_name = tag.get_str('tag') + commit = tag.get_str('commit') assert (tag_name, commit) == ('tag', repo.rev_parse('tag^{commit}')) checkout = os.path.join(str(tmpdir), 'checkout') @@ -1125,7 +1125,7 @@ def test_default_do_not_track_tags(cli, tmpdir, datafiles): result.assert_success() element = _yaml.load(element_path) - source = _yaml.node_get(element, dict, 'sources', indices=[0]) + source = element.get_sequence('sources').mapping_at(0) assert 'tags' not in source diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py index 800e0ced5..50bfe383c 100644 --- a/tests/sources/previous_source_access.py +++ b/tests/sources/previous_source_access.py @@ -24,7 +24,7 @@ def test_custom_transform_source(cli, datafiles): # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = _yaml.load(project_config_path) - aliases = _yaml.node_get(project_config, dict, "aliases") + aliases = project_config.get_mapping("aliases") _yaml.node_set(aliases, "project_dir", "file://{}".format(project)) _yaml.dump(project_config, project_config_path) diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py index bec4f9913..cd0f621d5 100644 --- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py +++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py @@ -32,7 +32,7 @@ class FooTransformSource(Source): def configure(self, node): self.node_validate(node, ['ref', *Source.COMMON_CONFIG_KEYS]) - self.ref = self.node_get_member(node, str, 'ref', None) + self.ref = node.get_str('ref', None) def preflight(self): pass diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py index 2e1f72138..6ea6cc3e8 100644 --- a/tests/testutils/__init__.py +++ b/tests/testutils/__init__.py @@ -28,5 +28,4 @@ from .element_generators import create_element_size, update_element_size from .junction import generate_junction from .runner_integration import wait_for_cache_granularity from .python_repo import setup_pypi_repo -from .yaml import yaml_file_get_provenance from .platform import override_platform_uname diff --git a/tests/testutils/yaml.py b/tests/testutils/yaml.py deleted file mode 100644 index a1dcb8646..000000000 --- a/tests/testutils/yaml.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Angelos Evripiotis <jevripiotis@bloomberg.net> - -from buildstream import _yaml - - -# yaml_file_get_provenance() -# -# Load a yaml file and return its _yaml.ProvenanceInformation object. -# -# This is useful for checking the provenance in BuildStream output is as -# expected. -# -# Args: -# path (str): The path to the file to be loaded -# shortname (str): How the path should appear in the error -# key (str): Optional key to look up in the loaded file -# indices (list of indexes): Optional index path, in the case of list values -# -# Returns: -# The ProvenanceInformation of the dict, member or list element -# -def yaml_file_get_provenance(path, shortname, key=None, indices=None): - file_node = _yaml.load(path, shortname) - if key: - required_node = _yaml.node_get(file_node, dict, key, indices=indices) - else: - required_node = file_node - provenance = _yaml.node_get_provenance(required_node) - assert provenance is not None - return provenance |
