summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBenjamin Schubert <contact@benschubert.me>2019-06-26 14:11:52 +0000
committerBenjamin Schubert <contact@benschubert.me>2019-06-26 14:11:52 +0000
commit024b1ddfadbb71c6f10caa553a10885827b9366f (patch)
treee4fed04f093eba0d09964f2e38e6249292fb0eed
parent950d7e90a2d124254093a0f05a9b80ab6a8ed95c (diff)
parent9d84e70482a94dfdefbe2a1bfb14fe43459d6324 (diff)
downloadbuildstream-024b1ddfadbb71c6f10caa553a10885827b9366f.tar.gz
Merge branch 'bschubert/node-api-noget' into 'bschubert/new-node-api'
introduce a Node api, and remove node_get See merge request BuildStream/buildstream!1401
-rwxr-xr-xdoc/bst2html.py24
-rw-r--r--src/buildstream/_basecache.py4
-rw-r--r--src/buildstream/_cas/casremote.py12
-rw-r--r--src/buildstream/_context.py50
-rw-r--r--src/buildstream/_frontend/cli.py2
-rw-r--r--src/buildstream/_gitsourcebase.py32
-rw-r--r--src/buildstream/_includes.py15
-rw-r--r--src/buildstream/_loader/loader.py30
-rw-r--r--src/buildstream/_loader/types.pyx22
-rw-r--r--src/buildstream/_options/option.py5
-rw-r--r--src/buildstream/_options/optionbool.py6
-rw-r--r--src/buildstream/_options/optionenum.py6
-rw-r--r--src/buildstream/_options/optionflags.py6
-rw-r--r--src/buildstream/_options/optionpool.py6
-rw-r--r--src/buildstream/_plugincontext.py13
-rw-r--r--src/buildstream/_project.py104
-rw-r--r--src/buildstream/_projectrefs.py12
-rw-r--r--src/buildstream/_variables.pyx4
-rw-r--r--src/buildstream/_workspaces.py13
-rw-r--r--src/buildstream/_yaml.pxd25
-rw-r--r--src/buildstream/_yaml.pyx379
-rw-r--r--src/buildstream/buildelement.py13
-rw-r--r--src/buildstream/element.py113
-rw-r--r--src/buildstream/plugin.py82
-rw-r--r--src/buildstream/plugins/elements/compose.py8
-rw-r--r--src/buildstream/plugins/elements/filter.py8
-rw-r--r--src/buildstream/plugins/elements/junction.py6
-rw-r--r--src/buildstream/plugins/elements/script.py5
-rw-r--r--src/buildstream/plugins/sources/_downloadablefilesource.py8
-rw-r--r--src/buildstream/plugins/sources/bzr.py8
-rw-r--r--src/buildstream/plugins/sources/deb.py2
-rw-r--r--src/buildstream/plugins/sources/patch.py2
-rw-r--r--src/buildstream/plugins/sources/pip.py12
-rw-r--r--src/buildstream/plugins/sources/remote.py4
-rw-r--r--src/buildstream/plugins/sources/tar.py3
-rw-r--r--src/buildstream/plugins/sources/zip.py3
-rw-r--r--src/buildstream/sandbox/_sandboxremote.py8
-rw-r--r--src/buildstream/source.py10
-rw-r--r--src/buildstream/testing/_sourcetests/mirror.py8
-rw-r--r--src/buildstream/testing/_sourcetests/track.py6
-rw-r--r--tests/elements/filter.py24
-rw-r--r--tests/elements/filter/basic/element_plugins/dynamic.py2
-rw-r--r--tests/format/include.py32
-rw-r--r--tests/format/include_composition.py22
-rw-r--r--tests/format/optionarch.py2
-rw-r--r--tests/format/optionbool.py4
-rw-r--r--tests/format/optioneltmask.py4
-rw-r--r--tests/format/optionenum.py4
-rw-r--r--tests/format/optionexports.py2
-rw-r--r--tests/format/optionflags.py4
-rw-r--r--tests/format/optionos.py2
-rw-r--r--tests/format/optionoverrides.py2
-rw-r--r--tests/format/options.py22
-rw-r--r--tests/format/project.py10
-rw-r--r--tests/format/projectoverrides.py2
-rw-r--r--tests/format/variables.py4
-rw-r--r--tests/frontend/buildcheckout.py7
-rw-r--r--tests/frontend/cross_junction_workspace.py15
-rw-r--r--tests/frontend/fetch.py7
-rw-r--r--tests/frontend/init.py22
-rw-r--r--tests/frontend/project/sources/fetch_source.py6
-rw-r--r--tests/frontend/show.py7
-rw-r--r--tests/frontend/track.py12
-rw-r--r--tests/frontend/workspace.py19
-rw-r--r--tests/internals/yaml.py68
-rw-r--r--tests/sources/git.py24
-rw-r--r--tests/sources/previous_source_access.py2
-rw-r--r--tests/sources/previous_source_access/plugins/sources/foo_transform.py2
-rw-r--r--tests/testutils/__init__.py1
-rw-r--r--tests/testutils/yaml.py47
70 files changed, 703 insertions, 757 deletions
diff --git a/doc/bst2html.py b/doc/bst2html.py
index 448ab1389..c6dcf3673 100755
--- a/doc/bst2html.py
+++ b/doc/bst2html.py
@@ -332,9 +332,9 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
#
if not force:
needs_build = False
- commands = _yaml.node_get(desc, list, 'commands')
+ commands = desc.get_sequence('commands')
for command in commands:
- output = _yaml.node_get(command, str, 'output', default_value=None)
+ output = command.get_str('output', default=None)
if output is not None and check_needs_build(desc_dir, output, force=False):
needs_build = True
break
@@ -347,7 +347,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
# tarball. This workaround lets us build docs from
# a source distribution tarball.
#
- symlinks = _yaml.node_get(desc, dict, 'workaround-symlinks', default_value={})
+ symlinks = desc.get_mapping('workaround-symlinks', default={})
for symlink, target in _yaml.node_items(symlinks):
# Resolve real path to where symlink should be
@@ -367,7 +367,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
# not a source distribution, no need to complain
pass
- remove_files = _yaml.node_get(desc, list, 'remove-files', default_value=[])
+ remove_files = desc.get_sequence('remove-files', default=[]).as_str_list()
for remove_file in remove_files:
remove_file = os.path.join(desc_dir, remove_file)
remove_file = os.path.realpath(remove_file)
@@ -379,23 +379,21 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
# Run commands
#
- commands = _yaml.node_get(desc, list, 'commands')
- for c in commands:
- command = _yaml.node_get(desc, dict, 'commands', indices=[commands.index(c)])
-
+ commands = desc.get_sequence('commands')
+ for command in commands:
# Get the directory where this command should be run
- directory = _yaml.node_get(command, str, 'directory')
+ directory = command.get_str('directory')
directory = os.path.join(desc_dir, directory)
directory = os.path.realpath(directory)
# Get the command string
- command_str = _yaml.node_get(command, str, 'command')
+ command_str = command.get_str('command')
# Check whether this is a shell command and not a bst command
- is_shell = _yaml.node_get(command, bool, 'shell', default_value=False)
+ is_shell = command.get_bool('shell', default=False)
# Check if there is fake output
- command_fake_output = _yaml.node_get(command, str, 'fake-output', default_value=None)
+ command_fake_output = command.get_str('fake-output', default=None)
# Run the command, or just use the fake output
if command_fake_output is None:
@@ -407,7 +405,7 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
command_out = command_fake_output
# Encode and save the output if that was asked for
- output = _yaml.node_get(command, str, 'output', default_value=None)
+ output = command.get_str('output', default=None)
if output is not None:
# Convert / Generate a nice <div>
converted = generate_html(command_out, directory, config_file,
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index ef9414787..e8923c7b1 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -74,10 +74,10 @@ class BaseCache():
cache_specs = []
try:
- artifacts = [_yaml.node_get(config_node, dict, cls.config_node_name)]
+ artifacts = [config_node.get_mapping(cls.config_node_name)]
except LoadError:
try:
- artifacts = _yaml.node_get(config_node, list, cls.config_node_name, default_value=[])
+ artifacts = config_node.get_sequence(cls.config_node_name, default=[])
except LoadError:
provenance = _yaml.node_get_provenance(config_node, key=cls.config_node_name)
raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py
index cd46e9c38..56cfea20f 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -32,24 +32,24 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
@staticmethod
def _new_from_config_node(spec_node, basedir=None):
_yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance-name'])
- url = _yaml.node_get(spec_node, str, 'url')
- push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
+ url = spec_node.get_str('url')
+ push = spec_node.get_bool('push', default=False)
if not url:
provenance = _yaml.node_get_provenance(spec_node, 'url')
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: empty artifact cache URL".format(provenance))
- instance_name = _yaml.node_get(spec_node, str, 'instance-name', default_value=None)
+ instance_name = spec_node.get_str('instance-name', default=None)
- server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None)
+ server_cert = spec_node.get_str('server-cert', default=None)
if server_cert and basedir:
server_cert = os.path.join(basedir, server_cert)
- client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None)
+ client_key = spec_node.get_str('client-key', default=None)
if client_key and basedir:
client_key = os.path.join(basedir, client_key)
- client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None)
+ client_cert = spec_node.get_str('client-cert', default=None)
if client_cert and basedir:
client_cert = os.path.join(basedir, client_cert)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 3f6e6ac3c..21c272daa 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -222,7 +222,7 @@ class Context():
# Allow the ~ tilde expansion and any environment variables in
# path specification in the config files.
#
- path = _yaml.node_get(defaults, str, directory)
+ path = defaults.get_str(directory)
path = os.path.expanduser(path)
path = os.path.expandvars(path)
path = os.path.normpath(path)
@@ -251,10 +251,10 @@ class Context():
# Load quota configuration
# We need to find the first existing directory in the path of our
# cachedir - the cachedir may not have been created yet.
- cache = _yaml.node_get(defaults, dict, 'cache')
+ cache = defaults.get_mapping('cache')
_yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees'])
- self.config_cache_quota_string = _yaml.node_get(cache, str, 'quota')
+ self.config_cache_quota_string = cache.get_str('quota')
try:
self.config_cache_quota = utils._parse_size(self.config_cache_quota_string,
self.casdir)
@@ -273,42 +273,43 @@ class Context():
self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
# Load pull build trees configuration
- self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
+ self.pull_buildtrees = cache.get_bool('pull-buildtrees')
# Load cache build trees configuration
self.cache_buildtrees = _node_get_option_str(
cache, 'cache-buildtrees', ['always', 'auto', 'never'])
# Load logging config
- logging = _yaml.node_get(defaults, dict, 'logging')
+ logging = defaults.get_mapping('logging')
_yaml.node_validate(logging, [
'key-length', 'verbose',
'error-lines', 'message-lines',
'debug', 'element-format', 'message-format'
])
- self.log_key_length = _yaml.node_get(logging, int, 'key-length')
- self.log_debug = _yaml.node_get(logging, bool, 'debug')
- self.log_verbose = _yaml.node_get(logging, bool, 'verbose')
- self.log_error_lines = _yaml.node_get(logging, int, 'error-lines')
- self.log_message_lines = _yaml.node_get(logging, int, 'message-lines')
- self.log_element_format = _yaml.node_get(logging, str, 'element-format')
- self.log_message_format = _yaml.node_get(logging, str, 'message-format')
+ self.log_key_length = logging.get_int('key-length')
+ self.log_debug = logging.get_bool('debug')
+ self.log_verbose = logging.get_bool('verbose')
+ self.log_error_lines = logging.get_int('error-lines')
+ self.log_message_lines = logging.get_int('message-lines')
+ self.log_message_lines = logging.get_int('message-lines')
+ self.log_element_format = logging.get_str('element-format')
+ self.log_message_format = logging.get_str('message-format')
# Load scheduler config
- scheduler = _yaml.node_get(defaults, dict, 'scheduler')
+ scheduler = defaults.get_mapping('scheduler')
_yaml.node_validate(scheduler, [
'on-error', 'fetchers', 'builders',
'pushers', 'network-retries'
])
self.sched_error_action = _node_get_option_str(
scheduler, 'on-error', ['continue', 'quit', 'terminate'])
- self.sched_fetchers = _yaml.node_get(scheduler, int, 'fetchers')
- self.sched_builders = _yaml.node_get(scheduler, int, 'builders')
- self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers')
- self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries')
+ self.sched_fetchers = scheduler.get_int('fetchers')
+ self.sched_builders = scheduler.get_int('builders')
+ self.sched_pushers = scheduler.get_int('pushers')
+ self.sched_network_retries = scheduler.get_int('network-retries')
# Load per-projects overrides
- self._project_overrides = _yaml.node_get(defaults, dict, 'projects', default_value={})
+ self._project_overrides = defaults.get_mapping('projects', default={})
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
@@ -398,17 +399,16 @@ class Context():
# get_overrides():
#
# Fetch the override dictionary for the active project. This returns
- # a node loaded from YAML and as such, values loaded from the returned
- # node should be loaded using the _yaml.node_get() family of functions.
+ # a node loaded from YAML.
#
# Args:
# project_name (str): The project name
#
# Returns:
- # (dict): The overrides dictionary for the specified project
+ # (MappingNode): The overrides dictionary for the specified project
#
def get_overrides(self, project_name):
- return _yaml.node_get(self._project_overrides, dict, project_name, default_value={})
+ return self._project_overrides.get_mapping(project_name, default={})
# get_strict():
#
@@ -423,7 +423,7 @@ class Context():
# so work out if we should be strict, and then cache the result
toplevel = self.get_toplevel_project()
overrides = self.get_overrides(toplevel.name)
- self._strict_build_plan = _yaml.node_get(overrides, bool, 'strict', default_value=True)
+ self._strict_build_plan = overrides.get_bool('strict', default=True)
# If it was set by the CLI, it overrides any config
# Ditto if we've already computed this, then we return the computed
@@ -740,7 +740,7 @@ class Context():
# _node_get_option_str()
#
-# Like _yaml.node_get(), but also checks value is one of the allowed option
+# Like Node.get_scalar().as_str(), but also checks value is one of the allowed option
# strings. Fetches a value from a dictionary node, and makes sure it's one of
# the pre-defined options.
#
@@ -756,7 +756,7 @@ class Context():
# LoadError, when the value is not of the expected type, or is not found.
#
def _node_get_option_str(node, key, allowed_options):
- result = _yaml.node_get(node, str, key)
+ result = node.get_str(key)
if result not in allowed_options:
provenance = _yaml.node_get_provenance(node, key)
raise LoadError(LoadErrorReason.INVALID_DATA,
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 53adb188c..f2f149403 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -100,7 +100,7 @@ def complete_target(args, incomplete):
return []
# The project is not required to have an element-path
- element_directory = _yaml.node_get(project, str, 'element-path', default_value='')
+ element_directory = project.get_str('element-path', default='')
# If a project was loaded, use its element-path to
# adjust our completion's base directory
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 1f539d820..4837877f4 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -376,25 +376,25 @@ class _GitSourceBase(Source):
BST_MIRROR_CLASS = _GitMirror
def configure(self, node):
- ref = self.node_get_member(node, str, 'ref', None)
+ ref = node.get_str('ref', None)
config_keys = ['url', 'track', 'ref', 'submodules',
'checkout-submodules', 'ref-format',
'track-tags', 'tags']
self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
- tags_node = self.node_get_member(node, list, 'tags', [])
+ tags_node = node.get_sequence('tags', [])
for tag_node in tags_node:
self.node_validate(tag_node, ['tag', 'commit', 'annotated'])
tags = self._load_tags(node)
- self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
+ self.track_tags = node.get_bool('track-tags', default=False)
- self.original_url = self.node_get_member(node, str, 'url')
+ self.original_url = node.get_str('url')
self.mirror = self.BST_MIRROR_CLASS(self, '', self.original_url, ref, tags=tags, primary=True)
- self.tracking = self.node_get_member(node, str, 'track', None)
+ self.tracking = node.get_str('track', None)
- self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
+ self.ref_format = node.get_str('ref-format', 'sha1')
if self.ref_format not in ['sha1', 'git-describe']:
provenance = self.node_provenance(node, member_name='ref-format')
raise SourceError("{}: Unexpected value for ref-format: {}".format(provenance, self.ref_format))
@@ -405,17 +405,17 @@ class _GitSourceBase(Source):
raise SourceError("{}: Git sources require a ref and/or track".format(self),
reason="missing-track-and-ref")
- self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
+ self.checkout_submodules = node.get_bool('checkout-submodules', default=True)
self.submodules = []
# Parse a dict of submodule overrides, stored in the submodule_overrides
# and submodule_checkout_overrides dictionaries.
self.submodule_overrides = {}
self.submodule_checkout_overrides = {}
- modules = self.node_get_member(node, dict, 'submodules', {})
+ modules = node.get_mapping('submodules', {})
for path, _ in self.node_items(modules):
- submodule = self.node_get_member(modules, dict, path)
- url = self.node_get_member(submodule, str, 'url', None)
+ submodule = modules.get_mapping(path)
+ url = submodule.get_str('url', None)
# Make sure to mark all URLs that are specified in the configuration
if url:
@@ -423,7 +423,7 @@ class _GitSourceBase(Source):
self.submodule_overrides[path] = url
if 'checkout' in submodule:
- checkout = self.node_get_member(submodule, bool, 'checkout')
+ checkout = submodule.get_bool('checkout')
self.submodule_checkout_overrides[path] = checkout
self.mark_download_url(self.original_url)
@@ -464,7 +464,7 @@ class _GitSourceBase(Source):
return Consistency.INCONSISTENT
def load_ref(self, node):
- self.mirror.ref = self.node_get_member(node, str, 'ref', None)
+ self.mirror.ref = node.get_str('ref', None)
self.mirror.tags = self._load_tags(node)
def get_ref(self):
@@ -663,11 +663,11 @@ class _GitSourceBase(Source):
def _load_tags(self, node):
tags = []
- tags_node = self.node_get_member(node, list, 'tags', [])
+ tags_node = node.get_sequence('tags', [])
for tag_node in tags_node:
- tag = self.node_get_member(tag_node, str, 'tag')
- commit_ref = self.node_get_member(tag_node, str, 'commit')
- annotated = self.node_get_member(tag_node, bool, 'annotated')
+ tag = tag_node.get_str('tag')
+ commit_ref = tag_node.get_str('commit')
+ annotated = tag_node.get_bool('annotated')
tags.append((tag, commit_ref, annotated))
return tags
diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py
index 8f507b566..2831c9765 100644
--- a/src/buildstream/_includes.py
+++ b/src/buildstream/_includes.py
@@ -35,17 +35,14 @@ class Includes:
if current_loader is None:
current_loader = self._loader
- includes = _yaml.node_get(node, None, '(@)', default_value=None)
- if isinstance(includes, str):
- includes = [includes]
+ includes_node = node.get_node('(@)', allowed_types=[_yaml.ScalarNode, _yaml.SequenceNode], allow_none=True)
- if not isinstance(includes, list) and includes is not None:
- provenance = _yaml.node_get_provenance(node, key='(@)')
- raise LoadError(LoadErrorReason.INVALID_DATA,
- "{}: {} must either be list or str".format(provenance, includes))
+ if includes_node:
+ if type(includes_node) is _yaml.ScalarNode: # pylint: disable=unidiomatic-typecheck
+ includes = [includes_node.as_str()]
+ else:
+ includes = includes_node.as_str_list()
- include_provenance = None
- if includes:
include_provenance = _yaml.node_get_provenance(node, key='(@)')
_yaml.node_del(node, '(@)')
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index fa3539b22..1bad5fe0b 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -227,7 +227,7 @@ class Loader():
message, detail=detail) from e
else:
raise
- kind = _yaml.node_get(node, str, Symbol.KIND)
+ kind = node.get_str(Symbol.KIND)
if kind == "junction":
self._first_pass_options.process_node(node)
else:
@@ -306,7 +306,7 @@ class Loader():
dep_deps = extract_depends_from_node(dep_element.node)
loader_queue.append((dep_element, list(reversed(dep_deps)), []))
- if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
+ if dep_element.node.get_str(Symbol.KIND) == 'junction':
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: Cannot depend on junction"
.format(dep.provenance))
@@ -466,33 +466,29 @@ class Loader():
elt_provenance = _yaml.node_get_provenance(node)
meta_sources = []
- sources = _yaml.node_get(node, list, Symbol.SOURCES, default_value=[])
- element_kind = _yaml.node_get(node, str, Symbol.KIND)
+ sources = node.get_sequence(Symbol.SOURCES, default=[])
+ element_kind = node.get_str(Symbol.KIND)
- # Safe loop calling into _yaml.node_get() for each element ensures
- # we have good error reporting
- for i in range(len(sources)):
- source = _yaml.node_get(node, dict, Symbol.SOURCES, indices=[i])
- kind = _yaml.node_get(source, str, Symbol.KIND)
+ for index, source in enumerate(sources):
+ kind = source.get_str(Symbol.KIND)
_yaml.node_del(source, Symbol.KIND)
# Directory is optional
- directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None)
+ directory = source.get_str(Symbol.DIRECTORY, default=None)
if directory:
_yaml.node_del(source, Symbol.DIRECTORY)
- index = sources.index(source)
meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
meta_sources.append(meta_source)
meta_element = MetaElement(self.project, element.name, element_kind,
elt_provenance, meta_sources,
- _yaml.node_get(node, dict, Symbol.CONFIG, default_value={}),
- _yaml.node_get(node, dict, Symbol.VARIABLES, default_value={}),
- _yaml.node_get(node, dict, Symbol.ENVIRONMENT, default_value={}),
- _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
- _yaml.node_get(node, dict, Symbol.PUBLIC, default_value={}),
- _yaml.node_get(node, dict, Symbol.SANDBOX, default_value={}),
+ node.get_mapping(Symbol.CONFIG, default={}),
+ node.get_mapping(Symbol.VARIABLES, default={}),
+ node.get_mapping(Symbol.ENVIRONMENT, default={}),
+ node.get_sequence(Symbol.ENV_NOCACHE, default=[]).as_str_list(),
+ node.get_mapping(Symbol.PUBLIC, default={}),
+ node.get_mapping(Symbol.SANDBOX, default={}),
element_kind == 'junction')
# Cache it now, make sure it's already there before recursing
diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx
index da33d6c54..dfd8f9046 100644
--- a/src/buildstream/_loader/types.pyx
+++ b/src/buildstream/_loader/types.pyx
@@ -72,12 +72,12 @@ cdef class Dependency:
self.provenance = provenance
- if type(dep) is str:
- self.name = <str> dep
+ if type(dep) is _yaml.ScalarNode:
+ self.name = dep.as_str()
self.dep_type = default_dep_type
self.junction = None
- elif type(dep) is _yaml.Node and type(dep.value) is dict:
+ elif type(dep) is _yaml.MappingNode:
if default_dep_type:
_yaml.node_validate(<_yaml.Node> dep, ['filename', 'junction'])
dep_type = default_dep_type
@@ -85,7 +85,7 @@ cdef class Dependency:
_yaml.node_validate(<_yaml.Node> dep, ['filename', 'type', 'junction'])
# Make type optional, for this we set it to None
- dep_type = <str> _yaml.node_get(<_yaml.Node> dep, str, <str> Symbol.TYPE, None, None)
+ dep_type = (<_yaml.MappingNode> dep).get_str(<str> Symbol.TYPE, None)
if dep_type is None or dep_type == <str> Symbol.ALL:
dep_type = None
elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
@@ -94,9 +94,9 @@ cdef class Dependency:
"{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
.format(provenance, dep_type))
- self.name = <str> _yaml.node_get(<_yaml.Node> dep, str, <str> Symbol.FILENAME)
+ self.name = (<_yaml.MappingNode> dep).get_str(<str> Symbol.FILENAME)
self.dep_type = dep_type
- self.junction = <str> _yaml.node_get(<_yaml.Node> dep, str, <str> Symbol.JUNCTION, None, None)
+ self.junction = (<_yaml.MappingNode> dep).get_str(<str> Symbol.JUNCTION, None)
else:
raise LoadError(LoadErrorReason.INVALID_DATA,
@@ -138,15 +138,15 @@ cdef class Dependency:
# acc (list): a list in which to add the loaded dependencies
#
cdef void _extract_depends_from_node(_yaml.Node node, str key, str default_dep_type, list acc) except *:
- cdef list depends = <list> _yaml.node_get(node, list, key, None, [])
- cdef int index
+ cdef _yaml.SequenceNode depends = node.get_sequence(key, [])
+ cdef _yaml.Node dep_node
cdef _yaml.ProvenanceInformation dep_provenance
- for index in range(len(depends)):
+ for dep_node in depends:
# FIXME: the provenance information would be obtainable from the Node directly if we stop
# stripping provenance and have proper nodes for str elements
- dep_provenance = <_yaml.ProvenanceInformation> _yaml.node_get_provenance(node, key=key, indices=[index])
- dependency = Dependency(depends[index], dep_provenance, default_dep_type=default_dep_type)
+ dep_provenance = <_yaml.ProvenanceInformation> _yaml.node_get_provenance(dep_node)
+ dependency = Dependency(dep_node, dep_provenance, default_dep_type=default_dep_type)
acc.append(dependency)
# Now delete the field, we dont want it anymore
diff --git a/src/buildstream/_options/option.py b/src/buildstream/_options/option.py
index ffdb4d272..6aa4cdc01 100644
--- a/src/buildstream/_options/option.py
+++ b/src/buildstream/_options/option.py
@@ -60,8 +60,9 @@ class Option():
# node (dict): The loaded YAML dictionary describing
# the option
def load(self, node):
- self.description = _yaml.node_get(node, str, 'description')
- self.variable = _yaml.node_get(node, str, 'variable', default_value=None)
+ self.description = node.get_str('description')
+ variable_node = node.get_str('variable', default=None)
+ self.variable = variable_node
# Assert valid symbol name for variable name
if self.variable is not None:
diff --git a/src/buildstream/_options/optionbool.py b/src/buildstream/_options/optionbool.py
index bdbb1d32a..1e5935e48 100644
--- a/src/buildstream/_options/optionbool.py
+++ b/src/buildstream/_options/optionbool.py
@@ -34,13 +34,13 @@ class OptionBool(Option):
super().load(node)
_yaml.node_validate(node, OPTION_SYMBOLS + ['default'])
- self.value = _yaml.node_get(node, bool, 'default')
+ self.value = node.get_bool('default')
def load_value(self, node, *, transform=None):
if transform:
- self.set_value(transform(_yaml.node_get(node, str, self.name)))
+ self.set_value(transform(node.get_str(self.name)))
else:
- self.value = _yaml.node_get(node, bool, self.name)
+ self.value = node.get_bool(self.name)
def set_value(self, value):
if value in ('True', 'true'):
diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py
index 889db965c..f04cecd8b 100644
--- a/src/buildstream/_options/optionenum.py
+++ b/src/buildstream/_options/optionenum.py
@@ -46,7 +46,7 @@ class OptionEnum(Option):
_yaml.node_validate(node, valid_symbols)
- self.values = _yaml.node_get(node, list, 'values', default_value=[])
+ self.values = node.get_sequence('values', default=[]).as_str_list()
if not self.values:
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: No values specified for {} option '{}'"
@@ -56,7 +56,7 @@ class OptionEnum(Option):
self.value = self.load_default_value(node)
def load_value(self, node, *, transform=None):
- self.value = _yaml.node_get(node, str, self.name)
+ self.value = node.get_str(self.name)
if transform:
self.value = transform(self.value)
self.validate(self.value, _yaml.node_get_provenance(node, self.name))
@@ -79,6 +79,6 @@ class OptionEnum(Option):
"Valid values: {}".format(", ".join(self.values)))
def load_default_value(self, node):
- value = _yaml.node_get(node, str, 'default')
+ value = node.get_str('default')
self.validate(value, _yaml.node_get_provenance(node, 'default'))
return value
diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py
index eba3a8dd5..c9758e403 100644
--- a/src/buildstream/_options/optionflags.py
+++ b/src/buildstream/_options/optionflags.py
@@ -53,11 +53,11 @@ class OptionFlags(Option):
"{}: No values specified for {} option '{}'"
.format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name))
- self.value = _yaml.node_get(node, list, 'default', default_value=[])
+ self.value = node.get_sequence('default', default=[]).as_str_list()
self.validate(self.value, _yaml.node_get_provenance(node, 'default'))
def load_value(self, node, *, transform=None):
- self.value = _yaml.node_get(node, list, self.name)
+ self.value = node.get_sequence(self.name).as_str_list()
if transform:
self.value = [transform(x) for x in self.value]
self.value = sorted(self.value)
@@ -90,4 +90,4 @@ class OptionFlags(Option):
def load_valid_values(self, node):
# Allow the more descriptive error to raise when no values
# exist rather than bailing out here (by specifying default_value)
- return _yaml.node_get(node, list, 'values', default_value=[])
+ return node.get_sequence('values', default=[]).as_str_list()
diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py
index de3af3e15..9a4a88c36 100644
--- a/src/buildstream/_options/optionpool.py
+++ b/src/buildstream/_options/optionpool.py
@@ -71,7 +71,7 @@ class OptionPool():
p = _yaml.node_get_provenance(options, option_name)
_yaml.assert_symbol_name(p, option_name, "option name", allow_dashes=False)
- opt_type_name = _yaml.node_get(option_definition, str, 'type')
+ opt_type_name = option_definition.get_str('type')
try:
opt_type = _OPTION_TYPES[opt_type_name]
except KeyError:
@@ -248,8 +248,8 @@ class OptionPool():
# Return true if a conditional was processed.
#
def _process_one_node(self, node):
- conditions = _yaml.node_get(node, list, '(?)', default_value=None)
- assertion = _yaml.node_get(node, str, '(!)', default_value=None)
+ conditions = node.get_sequence('(?)', default=None)
+ assertion = node.get_str('(!)', default=None)
# Process assersions first, we want to abort on the first encountered
# assertion in a given dictionary, and not lose an assertion due to
diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py
index 7a5407cf6..e95e881c4 100644
--- a/src/buildstream/_plugincontext.py
+++ b/src/buildstream/_plugincontext.py
@@ -22,7 +22,6 @@ import inspect
from ._exceptions import PluginError, LoadError, LoadErrorReason
from . import utils
-from . import _yaml
# A Context for loading plugin types
@@ -138,19 +137,19 @@ class PluginContext():
loaded_dependency = False
for origin in self._plugin_origins:
- if kind not in _yaml.node_get(origin, list, 'plugins'):
+ if kind not in origin.get_sequence('plugins').as_str_list():
continue
- if _yaml.node_get(origin, str, 'origin') == 'local':
- local_path = _yaml.node_get(origin, str, 'path')
+ if origin.get_str('origin') == 'local':
+ local_path = origin.get_str('path')
source = self._get_local_plugin_source(local_path)
- elif _yaml.node_get(origin, str, 'origin') == 'pip':
- package_name = _yaml.node_get(origin, str, 'package-name')
+ elif origin.get_str('origin') == 'pip':
+ package_name = origin.get_str('package-name')
source, defaults = self._get_pip_plugin_source(package_name, kind)
else:
raise PluginError("Failed to load plugin '{}': "
"Unexpected plugin origin '{}'"
- .format(kind, _yaml.node_get(origin, str, 'origin')))
+ .format(kind, origin.get_str('origin')))
loaded_dependency = True
break
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 114d25054..424af133e 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -84,7 +84,7 @@ class ProjectConfig:
self.source_overrides = {} # Source specific configurations
self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs.
self.default_mirror = None # The name of the preferred mirror.
- self._aliases = {} # Aliases dictionary
+ self._aliases = None # Aliases dictionary
# Project()
@@ -200,7 +200,7 @@ class Project():
if url and utils._ALIAS_SEPARATOR in url:
url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
- alias_url = _yaml.node_get(config._aliases, str, url_alias, default_value=None)
+ alias_url = config._aliases.get_str(url_alias, default=None)
if alias_url:
url = alias_url + url_body
@@ -264,7 +264,7 @@ class Project():
#
def get_path_from_node(self, node, key, *,
check_is_file=False, check_is_dir=False):
- path_str = _yaml.node_get(node, str, key)
+ path_str = node.get_str(key)
path = Path(path_str)
full_path = self._absolute_directory_path / path
@@ -404,7 +404,7 @@ class Project():
else:
config = self.config
- return _yaml.node_get(config._aliases, str, alias, default_value=None)
+ return config._aliases.get_str(alias, default=None)
# get_alias_uris()
#
@@ -419,7 +419,7 @@ class Project():
else:
config = self.config
- if not alias or alias not in config._aliases:
+ if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test
return [None]
mirror_list = []
@@ -429,7 +429,7 @@ class Project():
mirror_list = alias_mapping[alias] + mirror_list
else:
mirror_list += alias_mapping[alias]
- mirror_list.append(_yaml.node_get(config._aliases, str, alias))
+ mirror_list.append(config._aliases.get_str(alias))
return mirror_list
# load_elements()
@@ -573,7 +573,7 @@ class Project():
_yaml.composite(pre_config_node, self._project_conf)
# Assert project's format version early, before validating toplevel keys
- format_version = _yaml.node_get(pre_config_node, int, 'format-version')
+ format_version = pre_config_node.get_int('format-version')
if BST_FORMAT_VERSION < format_version:
major, minor = utils.get_bst_version()
raise LoadError(
@@ -585,7 +585,7 @@ class Project():
# The project name, element path and option declarations
# are constant and cannot be overridden by option conditional statements
- self.name = _yaml.node_get(self._project_conf, str, 'name')
+ self.name = self._project_conf.get_str('name')
# Validate that project name is a valid symbol name
_yaml.assert_symbol_name(_yaml.node_get_provenance(pre_config_node, 'name'),
@@ -600,12 +600,12 @@ class Project():
self.config.options = OptionPool(self.element_path)
self.first_pass_config.options = OptionPool(self.element_path)
- defaults = _yaml.node_get(pre_config_node, dict, 'defaults')
+ defaults = pre_config_node.get_mapping('defaults')
_yaml.node_validate(defaults, ['targets'])
- self._default_targets = _yaml.node_get(defaults, list, "targets")
+ self._default_targets = defaults.get_sequence("targets").as_str_list()
# Fatal warnings
- self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[])
+ self._fatal_warnings = pre_config_node.get_sequence('fatal-warnings', default=[]).as_str_list()
self.loader = Loader(self._context, self,
parent=parent_loader, fetch_subprojects=fetch_subprojects)
@@ -621,7 +621,7 @@ class Project():
ignore_unknown=True)
# Use separate file for storing source references
- self.ref_storage = _yaml.node_get(pre_config_node, str, 'ref-storage')
+ self.ref_storage = pre_config_node.get_str('ref-storage')
if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]:
p = _yaml.node_get_provenance(pre_config_node, 'ref-storage')
raise LoadError(LoadErrorReason.INVALID_DATA,
@@ -677,23 +677,20 @@ class Project():
self.remote_execution_specs = self._context.remote_execution_specs
# Load sandbox environment variables
- self.base_environment = _yaml.node_get(config, dict, 'environment')
- self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache')
+ self.base_environment = config.get_mapping('environment')
+ self.base_env_nocache = config.get_sequence('environment-nocache').as_str_list()
# Load sandbox configuration
- self._sandbox = _yaml.node_get(config, dict, 'sandbox')
+ self._sandbox = config.get_mapping('sandbox')
# Load project split rules
- self._splits = _yaml.node_get(config, dict, 'split-rules')
+ self._splits = config.get_mapping('split-rules')
# Support backwards compatibility for fail-on-overlap
- fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)
-
- if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap:
- self._fatal_warnings.append(CoreWarnings.OVERLAPS)
+ fail_on_overlap = config.get_scalar('fail-on-overlap', None)
# Deprecation check
- if fail_on_overlap is not None:
+ if not fail_on_overlap.is_none():
self._context.message(
Message(
None,
@@ -703,36 +700,37 @@ class Project():
)
)
+ if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap.as_bool():
+ self._fatal_warnings.append(CoreWarnings.OVERLAPS)
+
# Load project.refs if it exists, this may be ignored.
if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
self.refs.load(self.options)
# Parse shell options
- shell_options = _yaml.node_get(config, dict, 'shell')
+ shell_options = config.get_mapping('shell')
_yaml.node_validate(shell_options, ['command', 'environment', 'host-files'])
- self._shell_command = _yaml.node_get(shell_options, list, 'command')
+ self._shell_command = shell_options.get_sequence('command').as_str_list()
# Perform environment expansion right away
- shell_environment = _yaml.node_get(shell_options, dict, 'environment', default_value={})
+ shell_environment = shell_options.get_mapping('environment', default={})
for key in _yaml.node_keys(shell_environment):
- value = _yaml.node_get(shell_environment, str, key)
+ value = shell_environment.get_str(key)
self._shell_environment[key] = os.path.expandvars(value)
# Host files is parsed as a list for convenience
- host_files = _yaml.node_get(shell_options, list, 'host-files', default_value=[])
+ host_files = shell_options.get_sequence('host-files', default=[])
for host_file in host_files:
- if isinstance(host_file, str):
+ if isinstance(host_file, _yaml.ScalarNode):
mount = HostMount(host_file)
else:
# Some validation
- index = host_files.index(host_file)
- host_file_desc = _yaml.node_get(shell_options, dict, 'host-files', indices=[index])
- _yaml.node_validate(host_file_desc, ['path', 'host_path', 'optional'])
+ _yaml.node_validate(host_file, ['path', 'host_path', 'optional'])
# Parse the host mount
- path = _yaml.node_get(host_file_desc, str, 'path')
- host_path = _yaml.node_get(host_file_desc, str, 'host_path', default_value=None)
- optional = _yaml.node_get(host_file_desc, bool, 'optional', default_value=False)
+ path = host_file.get_str('path')
+ host_path = host_file.get_str('host_path', default=None)
+ optional = host_file.get_bool('optional', default=False)
mount = HostMount(path, host_path, optional)
self._shell_host_files.append(mount)
@@ -753,8 +751,8 @@ class Project():
# Element and Source type configurations will be composited later onto
# element/source types, so we delete it from here and run our final
# assertion after.
- output.element_overrides = _yaml.node_get(config, dict, 'elements', default_value={})
- output.source_overrides = _yaml.node_get(config, dict, 'sources', default_value={})
+ output.element_overrides = config.get_mapping('elements', default={})
+ output.source_overrides = config.get_mapping('sources', default={})
_yaml.node_del(config, 'elements', safe=True)
_yaml.node_del(config, 'sources', safe=True)
_yaml.node_final_assertions(config)
@@ -762,7 +760,7 @@ class Project():
self._load_plugin_factories(config, output)
# Load project options
- options_node = _yaml.node_get(config, dict, 'options', default_value={})
+ options_node = config.get_mapping('options', default={})
output.options.load(options_node)
if self.junction:
# load before user configuration
@@ -770,7 +768,7 @@ class Project():
# Collect option values specified in the user configuration
overrides = self._context.get_overrides(self.name)
- override_options = _yaml.node_get(overrides, dict, 'options', default_value={})
+ override_options = overrides.get_mapping('options', default={})
output.options.load_yaml_values(override_options)
if self._cli_options:
output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown)
@@ -789,7 +787,7 @@ class Project():
output.options.process_node(output.source_overrides)
# Load base variables
- output.base_variables = _yaml.node_get(config, dict, 'variables')
+ output.base_variables = config.get_mapping('variables')
# Add the project name as a default variable
_yaml.node_set(output.base_variables, 'project-name', self.name)
@@ -806,18 +804,18 @@ class Project():
output.options.export_variables(output.base_variables)
# Override default_mirror if not set by command-line
- output.default_mirror = self._default_mirror or _yaml.node_get(overrides, str,
- 'default-mirror', default_value=None)
+ output.default_mirror = self._default_mirror or overrides.get_str(
+ 'default-mirror', default=None)
- mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ mirrors = config.get_sequence('mirrors', default=[])
for mirror in mirrors:
allowed_mirror_fields = [
'name', 'aliases'
]
_yaml.node_validate(mirror, allowed_mirror_fields)
- mirror_name = _yaml.node_get(mirror, str, 'name')
+ mirror_name = mirror.get_str('name')
alias_mappings = {}
- for alias_mapping, uris in _yaml.node_items(_yaml.node_get(mirror, dict, 'aliases')):
+ for alias_mapping, uris in _yaml.node_items(mirror.get_mapping('aliases')):
assert isinstance(uris, list)
alias_mappings[alias_mapping] = list(uris)
output.mirrors[mirror_name] = alias_mappings
@@ -825,7 +823,7 @@ class Project():
output.default_mirror = mirror_name
# Source url aliases
- output._aliases = _yaml.node_get(config, dict, 'aliases', default_value={})
+ output._aliases = config.get_mapping('aliases', default={})
# _find_project_dir()
#
@@ -869,7 +867,7 @@ class Project():
plugin_element_origins = [] # Origins of custom elements
# Plugin origins and versions
- origins = _yaml.node_get(config, list, 'plugins', default_value=[])
+ origins = config.get_sequence('plugins', default=[])
source_format_versions = {}
element_format_versions = {}
for origin in origins:
@@ -880,7 +878,7 @@ class Project():
allowed_origins = ['core', 'local', 'pip']
_yaml.node_validate(origin, allowed_origin_fields)
- origin_value = _yaml.node_get(origin, str, 'origin')
+ origin_value = origin.get_str('origin')
if origin_value not in allowed_origins:
raise LoadError(
LoadErrorReason.INVALID_YAML,
@@ -888,26 +886,26 @@ class Project():
.format(origin_value))
# Store source versions for checking later
- source_versions = _yaml.node_get(origin, dict, 'sources', default_value={})
+ source_versions = origin.get_mapping('sources', default={})
for key in _yaml.node_keys(source_versions):
if key in source_format_versions:
raise LoadError(
LoadErrorReason.INVALID_YAML,
"Duplicate listing of source '{}'".format(key))
- source_format_versions[key] = _yaml.node_get(source_versions, int, key)
+ source_format_versions[key] = source_versions.get_int(key)
# Store element versions for checking later
- element_versions = _yaml.node_get(origin, dict, 'elements', default_value={})
+ element_versions = origin.get_mapping('elements', default={})
for key in _yaml.node_keys(element_versions):
if key in element_format_versions:
raise LoadError(
LoadErrorReason.INVALID_YAML,
"Duplicate listing of element '{}'".format(key))
- element_format_versions[key] = _yaml.node_get(element_versions, int, key)
+ element_format_versions[key] = element_versions.get_int(key)
# Store the origins if they're not 'core'.
# core elements are loaded by default, so storing is unnecessary.
- if _yaml.node_get(origin, str, 'origin') != 'core':
+ if origin.get_str('origin') != 'core':
self._store_origin(origin, 'sources', plugin_source_origins)
self._store_origin(origin, 'elements', plugin_element_origins)
@@ -941,13 +939,13 @@ class Project():
node_keys = [key for key in _yaml.node_keys(origin)]
if plugin_group in node_keys:
origin_node = _yaml.node_copy(origin)
- plugins = _yaml.node_get(origin, dict, plugin_group, default_value={})
+ plugins = origin.get_mapping(plugin_group, default={})
_yaml.node_set(origin_node, 'plugins', [k for k in _yaml.node_keys(plugins)])
for group in expected_groups:
if group in origin_node:
_yaml.node_del(origin_node, group)
- if _yaml.node_get(origin_node, str, 'origin') == 'local':
+ if origin_node.get_str('origin') == 'local':
path = self.get_path_from_node(origin, 'path',
check_is_dir=True)
# paths are passed in relative to the project, but must be absolute
diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py
index 09205a7c3..e72d4757e 100644
--- a/src/buildstream/_projectrefs.py
+++ b/src/buildstream/_projectrefs.py
@@ -123,8 +123,8 @@ class ProjectRefs():
def _lookup(self, toplevel, project, element, source_index, *, ensure=False):
# Fetch the project
try:
- projects = _yaml.node_get(toplevel, dict, 'projects')
- project_node = _yaml.node_get(projects, dict, project)
+ projects = toplevel.get_mapping('projects')
+ project_node = projects.get_mapping(project)
except LoadError:
if not ensure:
return None
@@ -133,16 +133,16 @@ class ProjectRefs():
# Fetch the element
try:
- element_list = _yaml.node_get(project_node, list, element)
+ element_list = project_node.get_sequence(element)
except LoadError:
if not ensure:
return None
- element_list = []
+ element_list = _yaml.new_empty_list_node()
_yaml.node_set(project_node, element, element_list)
# Fetch the source index
try:
- node = element_list[source_index]
+ node = element_list.mapping_at(source_index)
except IndexError:
if not ensure:
return None
@@ -150,6 +150,6 @@ class ProjectRefs():
# Pad the list with empty newly created dictionaries
_yaml.node_extend_list(project_node, element, source_index + 1, {})
- node = _yaml.node_get(project_node, dict, element, indices=[source_index])
+ node = project_node.get_sequence(element).mapping_at(source_index)
return node
diff --git a/src/buildstream/_variables.pyx b/src/buildstream/_variables.pyx
index 9b8b5a902..e8f083912 100644
--- a/src/buildstream/_variables.pyx
+++ b/src/buildstream/_variables.pyx
@@ -120,7 +120,7 @@ cdef class Variables:
# element, then override max-jobs to be 1.
# Initialize it as a string as all variables are processed as strings.
#
- if _yaml.node_get(node, bool, 'notparallel', None, False):
+ if node.get_bool('notparallel', False):
_yaml.node_set(node, 'max-jobs', str(1))
cdef dict ret = {}
@@ -128,7 +128,7 @@ cdef class Variables:
cdef str value
for key in _yaml.node_keys(node):
- value = <str> _yaml.node_get(node, str, key)
+ value = node.get_str(key)
ret[sys.intern(key)] = _parse_expstr(value)
return ret
diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py
index 9fbfb7e63..3847c7a85 100644
--- a/src/buildstream/_workspaces.py
+++ b/src/buildstream/_workspaces.py
@@ -570,7 +570,7 @@ class Workspaces():
#
def _parse_workspace_config(self, workspaces):
try:
- version = _yaml.node_get(workspaces, int, 'format-version', default_value=0)
+ version = workspaces.get_int('format-version', default=0)
except ValueError:
raise LoadError(LoadErrorReason.INVALID_DATA,
"Format version is not an integer in workspace configuration")
@@ -606,8 +606,7 @@ class Workspaces():
}
elif 1 <= version <= BST_WORKSPACE_FORMAT_VERSION:
- workspaces = _yaml.node_get(workspaces, dict, "workspaces",
- default_value=_yaml.new_empty_node())
+ workspaces = workspaces.get_mapping("workspaces", default={})
res = {element: self._load_workspace(node)
for element, node in _yaml.node_items(workspaces)}
@@ -631,11 +630,11 @@ class Workspaces():
#
def _load_workspace(self, node):
dictionary = {
- 'prepared': _yaml.node_get(node, bool, 'prepared', default_value=False),
- 'path': _yaml.node_get(node, str, 'path'),
- 'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
+ 'prepared': node.get_bool('prepared', default=False),
+ 'path': node.get_str('path'),
+ 'last_successful': node.get_str('last_successful', default=None),
'running_files': _yaml.node_sanitize(
- _yaml.node_get(node, dict, 'running_files', default_value=None),
+ node.get_mapping('running_files', default=None),
dict_type=dict),
}
return Workspace.from_dict(self._toplevel_project, dictionary)
diff --git a/src/buildstream/_yaml.pxd b/src/buildstream/_yaml.pxd
index 3cbad0d11..8deb17a78 100644
--- a/src/buildstream/_yaml.pxd
+++ b/src/buildstream/_yaml.pxd
@@ -28,6 +28,30 @@ cdef class Node:
cdef public int column
+cdef class MappingNode(Node):
+ cdef Node get(self, str key, default, default_constructor)
+ cpdef MappingNode get_mapping(self, str key, default=*)
+ cpdef Node get_node(self, str key, list allowed_types, bint allow_none=*)
+ cpdef ScalarNode get_scalar(self, str key, default=*)
+ cpdef SequenceNode get_sequence(self, str key, object default=*)
+ cpdef bint get_bool(self, str key, default=*) except *
+ cpdef int get_int(self, str key, default=*) except *
+ cpdef str get_str(self, str key, object default=*)
+
+
+cdef class ScalarNode(Node):
+ cpdef bint as_bool(self) except *
+ cpdef int as_int(self) except *
+ cpdef str as_str(self)
+ cpdef bint is_none(self)
+
+
+cdef class SequenceNode(Node):
+ cpdef MappingNode mapping_at(self, int index)
+ cpdef SequenceNode sequence_at(self, int index)
+ cpdef list as_str_list(self)
+
+
cdef class ProvenanceInformation:
cdef public Node node
@@ -39,7 +63,6 @@ cdef class ProvenanceInformation:
cpdef void node_del(Node node, str key, bint safe=*) except *
-cpdef object node_get(Node node, object expected_type, str key, list indices=*, object default_value=*, bint allow_none=*)
cpdef void node_validate(Node node, list valid_keys) except *
cpdef void node_set(Node node, object key, object value, list indices=*) except *
cpdef list node_keys(Node node)
diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx
index 678374272..4a49a9832 100644
--- a/src/buildstream/_yaml.pyx
+++ b/src/buildstream/_yaml.pyx
@@ -40,6 +40,11 @@ from ._exceptions import LoadError, LoadErrorReason
# pylint: disable=unidiomatic-typecheck
+# A sentinel to be used as a default argument for functions that need
+# to distinguish between a kwarg set to None and an unset kwarg.
+_sentinel = object()
+
+
# Node()
#
# Container for YAML loaded data and its provenance
@@ -70,6 +75,190 @@ cdef class Node:
return what in self.value
+cdef class ScalarNode(Node):
+
+ def __init__(self, object value, int file_index, int line, int column):
+ if type(value) is str:
+ value = value.strip()
+ self.value = value
+ self.file_index = file_index
+ self.line = line
+ self.column = column
+
+ cpdef bint is_none(self):
+ return self.value is None
+
+ cpdef bint as_bool(self) except *:
+ if type(self.value) is bool:
+ return self.value
+
+ # Don't coerce booleans to string, this makes "False" strings evaluate to True
+ if self.value in ('True', 'true'):
+ return True
+ elif self.value in ('False', 'false'):
+ return False
+ else:
+ provenance = node_get_provenance(self)
+ path = node_find_target(provenance.toplevel, self)[-1]
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type '{}'"
+ .format(provenance, path, bool.__name__, self.value))
+
+ cpdef int as_int(self) except *:
+ try:
+ return int(self.value)
+ except ValueError:
+ provenance = node_get_provenance(self)
+ path = node_find_target(provenance.toplevel, self)[-1]
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type '{}'"
+ .format(provenance, path, int.__name__))
+
+ cpdef str as_str(self):
+ # We keep 'None' as 'None' to simplify the API's usage and allow chaining for users
+ if self.value is None:
+ return None
+ return str(self.value)
+
+
+cdef class MappingNode(Node):
+
+ def __init__(self, dict value, int file_index, int line, int column):
+ self.value = value
+ self.file_index = file_index
+ self.line = line
+ self.column = column
+
+ cdef Node get(self, str key, object default, object default_constructor):
+ value = self.value.get(key, _sentinel)
+
+ if value is _sentinel:
+ if default is _sentinel:
+ provenance = node_get_provenance(self)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Dictionary did not contain expected key '{}'".format(provenance, key))
+
+ if default is None:
+ value = None
+ else:
+ value = default_constructor(default, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
+
+ return value
+
+ cpdef MappingNode get_mapping(self, str key, object default=_sentinel):
+ value = self.get(key, default, MappingNode)
+
+ if type(value) is not MappingNode and value is not None:
+ provenance = node_get_provenance(value)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type 'Mapping'"
+ .format(provenance, key))
+
+ return value
+
+ cpdef Node get_node(self, str key, list allowed_types, bint allow_none = False):
+ cdef value = self.value.get(key, _sentinel)
+
+ if value is _sentinel:
+ if allow_none:
+ return None
+
+ provenance = node_get_provenance(self)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Dictionary did not contain expected key '{}'".format(provenance, key))
+
+ if type(value) not in allowed_types:
+ provenance = node_get_provenance(self)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not one of the following: {}.".format(
+ provenance, key, ", ".join(allowed_types)))
+
+ return value
+
+ cpdef ScalarNode get_scalar(self, str key, object default=_sentinel):
+ value = self.get(key, default, ScalarNode)
+
+ if type(value) is not ScalarNode:
+ if value is None:
+ value = ScalarNode(None, self.file_index, 0, next_synthetic_counter())
+ else:
+ provenance = node_get_provenance(value)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type 'Scalar'"
+ .format(provenance, key))
+
+ return value
+
+ cpdef SequenceNode get_sequence(self, str key, object default=_sentinel):
+ value = self.get(key, default, SequenceNode)
+
+ if type(value) is not SequenceNode and value is not None:
+ provenance = node_get_provenance(value)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type 'Sequence'"
+ .format(provenance, key))
+
+ return value
+
+ cpdef bint get_bool(self, str key, object default=_sentinel) except *:
+ # TODO: don't go through 'get_scalar', we can directly get everything and optimize
+ cdef ScalarNode scalar = self.get_scalar(key, default)
+ return scalar.as_bool()
+
+ cpdef int get_int(self, str key, object default=_sentinel) except *:
+ # TODO: don't go through 'get_scalar', we can directly get everything and optimize
+ cdef ScalarNode scalar = self.get_scalar(key, default)
+ return scalar.as_int()
+
+ cpdef str get_str(self, str key, object default=_sentinel):
+ # TODO: don't go through 'get_scalar', we can directly get everything and optimize
+ cdef ScalarNode scalar = self.get_scalar(key, default)
+ return scalar.as_str()
+
+
+cdef class SequenceNode(Node):
+ def __init__(self, list value, int file_index, int line, int column):
+ self.value = value
+ self.file_index = file_index
+ self.line = line
+ self.column = column
+
+ cpdef MappingNode mapping_at(self, int index):
+ value = self.value[index]
+
+ if type(value) is not MappingNode:
+ provenance = node_get_provenance(self)
+ path = ["[{}]".format(p) for p in node_find_target(provenance.toplevel, self)] + ["[{}]".format(index)]
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type '{}'"
+ .format(provenance, path, MappingNode.__name__))
+ return value
+
+ cpdef SequenceNode sequence_at(self, int index):
+ value = self.value[index]
+
+ if type(value) is not SequenceNode:
+ provenance = node_get_provenance(self)
+ path = ["[{}]".format(p) for p in node_find_target(provenance.toplevel, self)] + ["[{}]".format(index)]
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type '{}'"
+ .format(provenance, path, SequenceNode.__name__))
+
+ return value
+
+ cpdef list as_str_list(self):
+ return [node.as_str() for node in self.value]
+
+ def __iter__(self):
+ return iter(self.value)
+
+ def __len__(self):
+ return len(self.value)
+
+ def __reversed__(self):
+ return reversed(self.value)
+
+
# Metadata container for a yaml toplevel node.
#
# This class contains metadata around a yaml node in order to be able
@@ -280,7 +469,7 @@ cdef class Representer:
return RepresenterState.doc
cdef RepresenterState _handle_doc_MappingStartEvent(self, object ev):
- newmap = Node({}, self._file_index, ev.start_mark.line, ev.start_mark.column)
+ newmap = MappingNode({}, self._file_index, ev.start_mark.line, ev.start_mark.column)
self.output.append(newmap)
return RepresenterState.wait_key
@@ -291,13 +480,13 @@ cdef class Representer:
cdef RepresenterState _handle_wait_value_ScalarEvent(self, object ev):
key = self.keys.pop()
(<dict> (<Node> self.output[-1]).value)[key] = \
- Node(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column)
+ ScalarNode(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column)
return RepresenterState.wait_key
cdef RepresenterState _handle_wait_value_MappingStartEvent(self, object ev):
cdef RepresenterState new_state = self._handle_doc_MappingStartEvent(ev)
key = self.keys.pop()
- (<dict> (<Node> self.output[-2]).value)[key] = self.output[-1]
+ (<dict> (<MappingNode> self.output[-2]).value)[key] = self.output[-1]
return new_state
cdef RepresenterState _handle_wait_key_MappingEndEvent(self, object ev):
@@ -313,13 +502,13 @@ cdef class Representer:
return RepresenterState.doc
cdef RepresenterState _handle_wait_value_SequenceStartEvent(self, object ev):
- self.output.append(Node([], self._file_index, ev.start_mark.line, ev.start_mark.column))
+ self.output.append(SequenceNode([], self._file_index, ev.start_mark.line, ev.start_mark.column))
(<dict> (<Node> self.output[-2]).value)[self.keys[-1]] = self.output[-1]
return RepresenterState.wait_list_item
cdef RepresenterState _handle_wait_list_item_SequenceStartEvent(self, object ev):
self.keys.append(len((<Node> self.output[-1]).value))
- self.output.append(Node([], self._file_index, ev.start_mark.line, ev.start_mark.column))
+ self.output.append(SequenceNode([], self._file_index, ev.start_mark.line, ev.start_mark.column))
(<list> (<Node> self.output[-2]).value).append(self.output[-1])
return RepresenterState.wait_list_item
@@ -336,7 +525,7 @@ cdef class Representer:
cdef RepresenterState _handle_wait_list_item_ScalarEvent(self, object ev):
(<Node> self.output[-1]).value.append(
- Node(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column))
+ ScalarNode(ev.value, self._file_index, ev.start_mark.line, ev.start_mark.column))
return RepresenterState.wait_list_item
cdef RepresenterState _handle_wait_list_item_MappingStartEvent(self, object ev):
@@ -353,6 +542,19 @@ cdef class Representer:
return RepresenterState.init
+cdef Node _create_node(object value, int file_index, int line, int column):
+ cdef type_value = type(value)
+
+ if type_value in [bool, str, type(None), int]:
+ return ScalarNode(value, file_index, line, column)
+ elif type_value is dict:
+ return MappingNode(value, file_index, line, column)
+ elif type_value is list:
+ return SequenceNode(value, file_index, line, column)
+ raise ValueError(
+ "Node values can only be 'list', 'dict', 'bool', 'str', 'int' or None. Not {}".format(type_value))
+
+
# Loads a dictionary from some YAML
#
# Args:
@@ -426,10 +628,10 @@ cpdef Node load_data(str data, int file_index=_SYNTHETIC_FILE_INDEX, str file_na
raise LoadError(LoadErrorReason.INVALID_YAML,
"Severely malformed YAML:\n\n{}\n\n".format(e)) from e
- if type(contents) != Node:
+ if type(contents) != MappingNode:
# Special case allowance for None, when the loaded file has only comments in it.
if contents is None:
- contents = Node({}, file_index, 0, 0)
+ contents = MappingNode({}, file_index, 0, 0)
else:
raise LoadError(LoadErrorReason.INVALID_YAML,
"YAML file has content of type '{}' instead of expected type 'dict': {}"
@@ -479,8 +681,6 @@ def dump(object contents, str filename=None):
# Returns: The Provenance of the dict, member or list element
#
cpdef ProvenanceInformation node_get_provenance(Node node, str key=None, list indices=None):
- assert type(node.value) is dict
-
if key is None:
# Retrieving the provenance for this node directly
return ProvenanceInformation(node)
@@ -495,105 +695,6 @@ cpdef ProvenanceInformation node_get_provenance(Node node, str key=None, list in
return ProvenanceInformation(nodeish)
-# A sentinel to be used as a default argument for functions that need
-# to distinguish between a kwarg set to None and an unset kwarg.
-_sentinel = object()
-
-
-# node_get()
-#
-# Fetches a value from a dictionary node and checks it for
-# an expected value. Use default_value when parsing a value
-# which is only optionally supplied.
-#
-# Args:
-# node (dict): The dictionary node
-# expected_type (type): The expected type for the value being searched
-# key (str): The key to get a value for in node
-# indices (list of ints): Optionally decend into lists of lists
-# default_value: Optionally return this value if the key is not found
-# allow_none: (bool): Allow None to be a valid value
-#
-# Returns:
-# The value if found in node, otherwise default_value is returned
-#
-# Raises:
-# LoadError, when the value found is not of the expected type
-#
-# Note:
-# Returned strings are stripped of leading and trailing whitespace
-#
-cpdef object node_get(Node node, object expected_type, str key, list indices=None, object default_value=_sentinel, bint allow_none=False):
- if indices is None:
- value = node.value.get(key, _sentinel)
-
- if value is _sentinel:
- if default_value is _sentinel:
- provenance = node_get_provenance(node)
- raise LoadError(LoadErrorReason.INVALID_DATA,
- "{}: Dictionary did not contain expected key '{}'".format(provenance, key))
-
- value = Node(default_value, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
- else:
- # Implied type check of the element itself
- # No need to synthesise useful node content as we destructure it immediately
- value = Node(node_get(node, list, key), _SYNTHETIC_FILE_INDEX, 0, 0)
- for index in indices:
- value = value.value[index]
- if type(value) is not Node:
- value = Node(value, _SYNTHETIC_FILE_INDEX, 0, 0)
-
- # Optionally allow None as a valid value for any type
- if value.value is None and (allow_none or default_value is None):
- return None
-
- if (expected_type is not None) and (type(value.value) is not expected_type):
- # Attempt basic conversions if possible, typically we want to
- # be able to specify numeric values and convert them to strings,
- # but we dont want to try converting dicts/lists
- try:
- if expected_type == bool and type(value.value) is str:
- # Dont coerce booleans to string, this makes "False" strings evaluate to True
- # We don't structure into full nodes since there's no need.
- if value.value in ('True', 'true'):
- value = Node(True, _SYNTHETIC_FILE_INDEX, 0, 0)
- elif value.value in ('False', 'false'):
- value = Node(False, _SYNTHETIC_FILE_INDEX, 0, 0)
- else:
- raise ValueError()
- elif not (expected_type == list or
- expected_type == dict or
- isinstance(value.value, (list, dict))):
- value = Node(expected_type(value.value), _SYNTHETIC_FILE_INDEX, 0, 0)
- else:
- raise ValueError()
- except (ValueError, TypeError):
- provenance = node_get_provenance(node, key=key, indices=indices)
- if indices:
- path = [key, *["[{:d}]".format(i) for i in indices]]
- path = "".join(path)
- else:
- path = key
- raise LoadError(LoadErrorReason.INVALID_DATA,
- "{}: Value of '{}' is not of the expected type '{}'"
- .format(provenance, path, expected_type.__name__))
-
- # Now collapse lists, and scalars, to their value, leaving nodes as-is
- if type(value.value) is not dict:
- value = value.value
-
- # Trim it at the bud, let all loaded strings from yaml be stripped of whitespace
- if type(value) is str:
- value = value.strip()
-
- elif type(value) is list:
- # Now we create a fresh list which unwraps the str and list types
- # semi-recursively.
- value = __trim_list_provenance(value)
-
- return value
-
-
cdef list __trim_list_provenance(list value):
cdef list ret = []
cdef Node entry
@@ -632,7 +733,7 @@ cpdef void node_set(Node node, object key, object value, list indices=None) exce
key = indices.pop()
for idx in indices:
node = <Node> (<list> node.value)[idx]
- if type(value) is Node:
+ if type(value) in [Node, MappingNode, ScalarNode, SequenceNode]:
node.value[key] = value
else:
try:
@@ -640,10 +741,11 @@ cpdef void node_set(Node node, object key, object value, list indices=None) exce
old_value = <Node> node.value[key]
except KeyError:
old_value = None
+
if old_value is None:
- node.value[key] = Node(value, node.file_index, node.line, next_synthetic_counter())
+ node.value[key] = _create_node(value, node.file_index, node.line, next_synthetic_counter())
else:
- node.value[key] = Node(value, old_value.file_index, old_value.line, old_value.column)
+ node.value[key] = _create_node(value, old_value.file_index, old_value.line, old_value.column)
# node_extend_list()
@@ -666,7 +768,7 @@ def node_extend_list(Node node, str key, Py_ssize_t length, object default):
cdef Node list_node = <Node> node.value.get(key)
if list_node is None:
- list_node = node.value[key] = Node([], node.file_index, node.line, next_synthetic_counter())
+ list_node = node.value[key] = SequenceNode([], node.file_index, node.line, next_synthetic_counter())
cdef list the_list = list_node.value
def_type = type(default)
@@ -687,7 +789,7 @@ def node_extend_list(Node node, str key, Py_ssize_t length, object default):
line_num += 1
- the_list.append(Node(value, file_index, line_num, next_synthetic_counter()))
+ the_list.append(_create_node(value, file_index, line_num, next_synthetic_counter()))
# node_items()
@@ -763,9 +865,9 @@ cpdef void node_del(Node node, str key, bint safe=False) except *:
def is_node(maybenode):
# It's a programming error to give this a Node which isn't a mapping
# so assert that.
- assert (type(maybenode) is not Node) or (type(maybenode.value) is dict)
+ assert (type(maybenode) not in [ScalarNode, SequenceNode])
# Now return the type check
- return type(maybenode) is Node
+ return type(maybenode) is MappingNode
# new_synthetic_file()
@@ -784,7 +886,7 @@ def is_node(maybenode):
#
def new_synthetic_file(str filename, object project=None):
cdef Py_ssize_t file_index = len(_FILE_LIST)
- cdef Node node = Node({}, file_index, 0, 0)
+ cdef Node node = MappingNode({}, file_index, 0, 0)
_FILE_LIST.append(FileInfo(filename,
filename,
@@ -804,9 +906,14 @@ def new_synthetic_file(str filename, object project=None):
#
def new_empty_node(Node ref_node=None):
if ref_node is not None:
- return Node({}, ref_node.file_index, ref_node.line, next_synthetic_counter())
+ return MappingNode({}, ref_node.file_index, ref_node.line, next_synthetic_counter())
else:
- return Node({}, _SYNTHETIC_FILE_INDEX, 0, 0)
+ return MappingNode({}, _SYNTHETIC_FILE_INDEX, 0, 0)
+
+
+# FIXME: we should never need that
+def new_empty_list_node():
+ return SequenceNode([], _SYNTHETIC_FILE_INDEX, 0, 0)
# new_node_from_dict()
@@ -827,8 +934,8 @@ cpdef Node new_node_from_dict(dict indict):
elif vtype is list:
ret[k] = __new_node_from_list(v)
else:
- ret[k] = Node(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
- return Node(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
+ ret[k] = ScalarNode(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
+ return MappingNode(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
# Internal function to help new_node_from_dict() to handle lists
@@ -841,8 +948,8 @@ cdef Node __new_node_from_list(list inlist):
elif vtype is list:
ret.append(__new_node_from_list(v))
else:
- ret.append(Node(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()))
- return Node(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
+ ret.append(ScalarNode(str(v), _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter()))
+ return SequenceNode(ret, _SYNTHETIC_FILE_INDEX, 0, next_synthetic_counter())
# _is_composite_list
@@ -1000,7 +1107,7 @@ cpdef void composite_dict(Node target, Node source, list path=None) except *:
if k not in target.value:
# Target lacks a dict at that point, make a fresh one with
# the same provenance as the incoming dict
- target.value[k] = Node({}, v.file_index, v.line, v.column)
+ target.value[k] = MappingNode({}, v.file_index, v.line, v.column)
if type(target.value) is not dict:
raise CompositeError(path,
"{}: Cannot compose dictionary onto {}".format(
@@ -1020,7 +1127,7 @@ cpdef void composite_dict(Node target, Node source, list path=None) except *:
# Like composite_dict(), but raises an all purpose LoadError for convenience
#
-cpdef void composite(Node target, Node source) except *:
+cpdef void composite(MappingNode target, MappingNode source) except *:
assert type(source.value) is dict
assert type(target.value) is dict
@@ -1040,7 +1147,7 @@ cpdef void composite(Node target, Node source) except *:
# Like composite(target, source), but where target overrides source instead.
#
-def composite_and_move(Node target, Node source):
+def composite_and_move(MappingNode target, MappingNode source):
composite(source, target)
cdef str key
@@ -1067,10 +1174,14 @@ cpdef object node_sanitize(object node, object dict_type=OrderedDict):
node_type = type(node)
# If we have an unwrappable node, unwrap it
- if node_type is Node:
+ # FIXME: we should only ever have Nodes here
+ if node_type in [MappingNode, SequenceNode]:
node = node.value
node_type = type(node)
+ if node_type is ScalarNode:
+ return node.value
+
# Short-circuit None which occurs ca. twice per element
if node is None:
return node
@@ -1157,7 +1268,7 @@ __NODE_ASSERT_COMPOSITION_DIRECTIVES = ('(>)', '(<)', '(=)')
# Returns:
# (Node): A deep copy of source with provenance preserved.
#
-cpdef Node node_copy(Node source):
+cpdef MappingNode node_copy(MappingNode source):
cdef dict copy = {}
cdef str key
cdef Node value
@@ -1173,7 +1284,7 @@ cpdef Node node_copy(Node source):
else:
raise ValueError("Unable to be quick about node_copy of {}".format(value_type))
- return Node(copy, source.file_index, source.line, source.column)
+ return MappingNode(copy, source.file_index, source.line, source.column)
# Internal function to help node_copy() but for lists.
@@ -1193,7 +1304,7 @@ cdef Node _list_copy(Node source):
else:
raise ValueError("Unable to be quick about list_copy of {}".format(item_type))
- return Node(copy, source.file_index, source.line, source.column)
+ return SequenceNode(copy, source.file_index, source.line, source.column)
# node_final_assertions()
@@ -1207,7 +1318,7 @@ cdef Node _list_copy(Node source):
# Raises:
# (LoadError): If any assertions fail
#
-cpdef void node_final_assertions(Node node) except *:
+cpdef void node_final_assertions(MappingNode node) except *:
cdef str key
cdef Node value
@@ -1303,7 +1414,7 @@ def assert_symbol_name(ProvenanceInformation provenance, str symbol_name, str pu
#
# Returns:
# (list): A path from `node` to `target` or None if `target` is not in the subtree
-cpdef list node_find_target(Node node, Node target, str key=None):
+cpdef list node_find_target(MappingNode node, Node target, str key=None):
if key is not None:
target = target.value[key]
@@ -1317,7 +1428,7 @@ cpdef list node_find_target(Node node, Node target, str key=None):
# Helper for node_find_target() which walks a value
-cdef bint _walk_find_target(Node node, list path, Node target):
+cdef bint _walk_find_target(Node node, list path, Node target) except *:
if node.file_index == target.file_index and node.line == target.line and node.column == target.column:
return True
elif type(node.value) is dict:
@@ -1341,7 +1452,7 @@ cdef bint _walk_list_node(Node node, list path, Node target):
# Helper for node_find_target() which walks a mapping
-cdef bint _walk_dict_node(Node node, list path, Node target):
+cdef bint _walk_dict_node(MappingNode node, list path, Node target):
cdef str k
cdef Node v
diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py
index 158f5fc11..48be1b83a 100644
--- a/src/buildstream/buildelement.py
+++ b/src/buildstream/buildelement.py
@@ -281,14 +281,11 @@ class BuildElement(Element):
# Private Local Methods #
#############################################################
def __get_commands(self, node, name):
- list_node = self.node_get_member(node, list, name, [])
- commands = []
-
- for i in range(len(list_node)):
- command = self.node_subst_list_element(node, name, [i])
- commands.append(command)
-
- return commands
+ raw_commands = node.get_sequence(name, []).as_str_list()
+ return [
+ self.substitute_variables(command)
+ for command in raw_commands
+ ]
def __run_command(self, sandbox, cmd):
# Note the -e switch to 'sh' means to exit with an error
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 60aefecad..3029c98cd 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -485,6 +485,9 @@ class Element(Plugin):
return None
+ def substitute_variables(self, value):
+ return self.__variables.subst(value)
+
def node_subst_member(self, node, member_name, default=_yaml._sentinel):
"""Fetch the value of a string node member, substituting any variables
in the loaded value with the element contextual variables.
@@ -500,10 +503,6 @@ class Element(Plugin):
Raises:
:class:`.LoadError`: When *member_name* is not found and no *default* was provided
- This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_member`
- except that it assumes the expected type is a string and will also perform variable
- substitutions.
-
**Example:**
.. code:: python
@@ -512,7 +511,7 @@ class Element(Plugin):
# variables in the returned string
name = self.node_subst_member(node, 'name')
"""
- value = self.node_get_member(node, str, member_name, default)
+ value = node.get_str(member_name, default)
try:
return self.__variables.subst(value)
except LoadError as e:
@@ -532,61 +531,16 @@ class Element(Plugin):
Raises:
:class:`.LoadError`
- This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_member`
- except that it assumes the expected type is a list of strings and will also
- perform variable substitutions.
"""
- value = self.node_get_member(node, list, member_name)
ret = []
- for index, x in enumerate(value):
+ for value in node.get_sequence(member_name):
try:
- ret.append(self.__variables.subst(x))
+ ret.append(self.__variables.subst(value.as_str()))
except LoadError as e:
- provenance = _yaml.node_get_provenance(node, key=member_name, indices=[index])
+ provenance = _yaml.node_get_provenance(value)
raise LoadError(e.reason, '{}: {}'.format(provenance, e), detail=e.detail) from e
return ret
- def node_subst_list_element(self, node, member_name, indices):
- """Fetch the value of a list element from a node member, substituting any variables
- in the loaded value with the element contextual variables.
-
- Args:
- node (dict): A dictionary loaded from YAML
- member_name (str): The name of the member to fetch
- indices (list of int): List of indices to search, in case of nested lists
-
- Returns:
- The value of the list element in *member_name* at the specified *indices*
-
- Raises:
- :class:`.LoadError`
-
- This is essentially the same as :func:`~buildstream.plugin.Plugin.node_get_list_element`
- except that it assumes the expected type is a string and will also perform variable
- substitutions.
-
- **Example:**
-
- .. code:: python
-
- # Fetch the list itself
- strings = self.node_get_member(node, list, 'strings')
-
- # Iterate over the list indices
- for i in range(len(strings)):
-
- # Fetch the strings in this list, substituting content
- # with our element's variables if needed
- string = self.node_subst_list_element(
- node, 'strings', [ i ])
- """
- value = self.node_get_list_element(node, str, member_name, indices)
- try:
- return self.__variables.subst(value)
- except LoadError as e:
- provenance = _yaml.node_get_provenance(node, key=member_name, indices=indices)
- raise LoadError(e.reason, '{}: {}'.format(provenance, e), detail=e.detail) from e
-
def compute_manifest(self, *, include=None, exclude=None, orphans=True):
"""Compute and return this element's selective manifest
@@ -854,9 +808,9 @@ class Element(Plugin):
if bstdata is not None:
with sandbox.batch(SandboxFlags.NONE):
- commands = self.node_get_member(bstdata, list, 'integration-commands', [])
- for i in range(len(commands)):
- cmd = self.node_subst_list_element(bstdata, 'integration-commands', [i])
+ commands = bstdata.get_sequence('integration-commands', []).as_str_list()
+ for command in commands:
+ cmd = self.substitute_variables(command)
sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/',
label=cmd)
@@ -896,7 +850,7 @@ class Element(Plugin):
if self.__dynamic_public is None:
self.__load_public_data()
- data = _yaml.node_get(self.__dynamic_public, dict, domain, default_value=None)
+ data = self.__dynamic_public.get_mapping(domain, default=None)
if data is not None:
data = _yaml.node_copy(data)
@@ -2539,9 +2493,9 @@ class Element(Plugin):
@classmethod
def __compose_default_splits(cls, project, defaults, is_junction):
- element_public = _yaml.node_get(defaults, dict, 'public', default_value={})
- element_bst = _yaml.node_get(element_public, dict, 'bst', default_value={})
- element_splits = _yaml.node_get(element_bst, dict, 'split-rules', default_value={})
+ element_public = defaults.get_mapping("public", default={})
+ element_bst = element_public.get_mapping("bst", default={})
+ element_splits = element_bst.get_mapping("split-rules", default={})
if is_junction:
splits = _yaml.node_copy(element_splits)
@@ -2581,7 +2535,7 @@ class Element(Plugin):
else:
elements = project.element_overrides
- overrides = _yaml.node_get(elements, dict, kind, default_value=None)
+ overrides = elements.get_mapping(kind, default=None)
if overrides:
_yaml.composite(defaults, overrides)
@@ -2593,7 +2547,7 @@ class Element(Plugin):
#
@classmethod
def __extract_environment(cls, project, meta):
- default_env = _yaml.node_get(cls.__defaults, dict, 'environment', default_value={})
+ default_env = cls.__defaults.get_mapping("environment", default={})
if meta.is_junction:
environment = _yaml.new_empty_node()
@@ -2624,7 +2578,7 @@ class Element(Plugin):
else:
project_nocache = project.base_env_nocache
- default_nocache = _yaml.node_get(cls.__defaults, list, 'environment-nocache', default_value=[])
+ default_nocache = cls.__defaults.get_sequence('environment-nocache', default=[]).as_str_list()
element_nocache = meta.env_nocache
# Accumulate values from the element default, the project and the element
@@ -2639,8 +2593,7 @@ class Element(Plugin):
#
@classmethod
def __extract_variables(cls, project, meta):
- default_vars = _yaml.node_get(cls.__defaults, dict, 'variables',
- default_value={})
+ default_vars = cls.__defaults.get_mapping('variables', default={})
if meta.is_junction:
variables = _yaml.node_copy(project.first_pass_config.base_variables)
@@ -2667,7 +2620,7 @@ class Element(Plugin):
def __extract_config(cls, meta):
# The default config is already composited with the project overrides
- config = _yaml.node_get(cls.__defaults, dict, 'config', default_value={})
+ config = cls.__defaults.get_mapping('config', default={})
config = _yaml.node_copy(config)
_yaml.composite(config, meta.config)
@@ -2693,7 +2646,7 @@ class Element(Plugin):
host_os = platform.get_host_os()
# The default config is already composited with the project overrides
- sandbox_defaults = _yaml.node_get(cls.__defaults, dict, 'sandbox', default_value={})
+ sandbox_defaults = cls.__defaults.get_mapping('sandbox', default={})
sandbox_defaults = _yaml.node_copy(sandbox_defaults)
_yaml.composite(sandbox_config, sandbox_defaults)
@@ -2703,16 +2656,16 @@ class Element(Plugin):
# Sandbox config, unlike others, has fixed members so we should validate them
_yaml.node_validate(sandbox_config, ['build-uid', 'build-gid', 'build-os', 'build-arch'])
- build_arch = _yaml.node_get(sandbox_config, str, 'build-arch', default_value=None)
+ build_arch = sandbox_config.get_str('build-arch', default=None)
if build_arch:
build_arch = Platform.canonicalize_arch(build_arch)
else:
build_arch = host_arch
return SandboxConfig(
- _yaml.node_get(sandbox_config, int, 'build-uid'),
- _yaml.node_get(sandbox_config, int, 'build-gid'),
- _yaml.node_get(sandbox_config, str, 'build-os', default_value=host_os),
+ sandbox_config.get_int('build-uid'),
+ sandbox_config.get_int('build-gid'),
+ sandbox_config.get_str('build-os', default=host_os),
build_arch)
# This makes a special exception for the split rules, which
@@ -2720,15 +2673,15 @@ class Element(Plugin):
#
@classmethod
def __extract_public(cls, meta):
- base_public = _yaml.node_get(cls.__defaults, dict, 'public', default_value={})
+ base_public = cls.__defaults.get_mapping('public', default={})
base_public = _yaml.node_copy(base_public)
- base_bst = _yaml.node_get(base_public, dict, 'bst', default_value={})
- base_splits = _yaml.node_get(base_bst, dict, 'split-rules', default_value={})
+ base_bst = base_public.get_mapping('bst', default={})
+ base_splits = base_bst.get_mapping('split-rules', default={})
element_public = _yaml.node_copy(meta.public)
- element_bst = _yaml.node_get(element_public, dict, 'bst', default_value={})
- element_splits = _yaml.node_get(element_bst, dict, 'split-rules', default_value={})
+ element_bst = element_public.get_mapping('bst', default={})
+ element_splits = element_bst.get_mapping('split-rules', default={})
# Allow elements to extend the default splits defined in their project or
# element specific defaults
@@ -2743,8 +2696,8 @@ class Element(Plugin):
# Expand the splits in the public data using the Variables in the element
def __expand_splits(self, element_public):
- element_bst = _yaml.node_get(element_public, dict, 'bst', default_value={})
- element_splits = _yaml.node_get(element_bst, dict, 'split-rules', default_value={})
+ element_bst = element_public.get_mapping('bst', default={})
+ element_splits = element_bst.get_mapping('split-rules', default={})
# Resolve any variables in the public split rules directly
for domain, splits in self.node_items(element_splits):
@@ -2758,7 +2711,7 @@ class Element(Plugin):
def __init_splits(self):
bstdata = self.get_public_data('bst')
- splits = self.node_get_member(bstdata, dict, 'split-rules')
+ splits = bstdata.get_mapping('split-rules')
self.__splits = {
domain: re.compile('^(?:' + '|'.join([utils._glob2re(r) for r in rules]) + ')$')
for domain, rules in self.node_items(splits)
@@ -2862,7 +2815,7 @@ class Element(Plugin):
# If this ever changes, things will go wrong unexpectedly.
if not self.__whitelist_regex:
bstdata = self.get_public_data('bst')
- whitelist = _yaml.node_get(bstdata, list, 'overlap-whitelist', default_value=[])
+ whitelist = bstdata.get_sequence('overlap-whitelist', default=[]).as_str_list()
whitelist_expressions = [utils._glob2re(self.__variables.subst(exp.strip())) for exp in whitelist]
expression = ('^(?:' + '|'.join(whitelist_expressions) + ')$')
self.__whitelist_regex = re.compile(expression)
diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py
index a12ff61ec..f2bd90b35 100644
--- a/src/buildstream/plugin.py
+++ b/src/buildstream/plugin.py
@@ -280,13 +280,7 @@ class Plugin():
Plugin implementors should implement this method to read configuration
data and store it.
- Plugins should use the :func:`Plugin.node_get_member() <buildstream.plugin.Plugin.node_get_member>`
- and :func:`Plugin.node_get_list_element() <buildstream.plugin.Plugin.node_get_list_element>`
- methods to fetch values from the passed `node`. This will ensure that a nice human readable error
- message will be raised if the expected configuration is not found, indicating the filename,
- line and column numbers.
-
- Further the :func:`Plugin.node_validate() <buildstream.plugin.Plugin.node_validate>` method
+ The :func:`Plugin.node_validate() <buildstream.plugin.Plugin.node_validate>` method
should be used to ensure that the user has not specified keys in `node` which are unsupported
by the plugin.
@@ -294,8 +288,7 @@ class Plugin():
For Elements, when variable substitution is desirable, the
:func:`Element.node_subst_member() <buildstream.element.Element.node_subst_member>`
- and :func:`Element.node_subst_list_element() <buildstream.element.Element.node_subst_list_element>`
- methods can be used.
+ method can be used.
"""
raise ImplError("{tag} plugin '{kind}' does not implement configure()".format(
tag=self.__type_tag, kind=self.get_kind()))
@@ -387,38 +380,6 @@ class Plugin():
provenance = _yaml.node_get_provenance(node, key=member_name)
return str(provenance)
- def node_get_member(self, node, expected_type, member_name, default=_yaml._sentinel, *, allow_none=False):
- """Fetch the value of a node member, raising an error if the value is
- missing or incorrectly typed.
-
- Args:
- node (Node): A dictionary loaded from YAML
- expected_type (type): The expected type of the node member
- member_name (str): The name of the member to fetch
- default (expected_type): A value to return when *member_name* is not specified in *node*
- allow_none (bool): Allow explicitly set None values in the YAML (*Since: 1.4*)
-
- Returns:
- The value of *member_name* in *node*, otherwise *default*
-
- Raises:
- :class:`.LoadError`: When *member_name* is not found and no *default* was provided
-
- Note:
- Returned strings are stripped of leading and trailing whitespace
-
- **Example:**
-
- .. code:: python
-
- # Expect a string 'name' in 'node'
- name = self.node_get_member(node, str, 'name')
-
- # Fetch an optional integer
- level = self.node_get_member(node, int, 'level', -1)
- """
- return _yaml.node_get(node, expected_type, member_name, default_value=default, allow_none=allow_none)
-
def node_set_member(self, node, key, value):
"""Set the value of a node member
Args:
@@ -530,41 +491,6 @@ class Plugin():
"""
_yaml.node_validate(node, valid_keys)
- def node_get_list_element(self, node, expected_type, member_name, indices):
- """Fetch the value of a list element from a node member, raising an error if the
- value is incorrectly typed.
-
- Args:
- node (dict): A dictionary loaded from YAML
- expected_type (type): The expected type of the node member
- member_name (str): The name of the member to fetch
- indices (list of int): List of indices to search, in case of nested lists
-
- Returns:
- The value of the list element in *member_name* at the specified *indices*
-
- Raises:
- :class:`.LoadError`
-
- Note:
- Returned strings are stripped of leading and trailing whitespace
-
- **Example:**
-
- .. code:: python
-
- # Fetch the list itself
- things = self.node_get_member(node, list, 'things')
-
- # Iterate over the list indices
- for i in range(len(things)):
-
- # Fetch dict things
- thing = self.node_get_list_element(
- node, dict, 'things', [ i ])
- """
- return _yaml.node_get(node, expected_type, member_name, indices=indices)
-
def debug(self, brief, *, detail=None):
"""Print a debugging message
@@ -894,10 +820,10 @@ class Plugin():
project = self.__project
for key, value in self.node_items(project.element_overrides):
- if _yaml.node_get(value, bool, 'suppress-deprecation-warnings', default_value=False):
+ if value.get_bool('suppress-deprecation-warnings', default=False):
silenced_warnings.add(key)
for key, value in self.node_items(project.source_overrides):
- if _yaml.node_get(value, bool, 'suppress-deprecation-warnings', default_value=False):
+ if value.get_bool('suppress-deprecation-warnings', default=False):
silenced_warnings.add(key)
return self.get_kind() in silenced_warnings
diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py
index b672cde0c..a9f18875f 100644
--- a/src/buildstream/plugins/elements/compose.py
+++ b/src/buildstream/plugins/elements/compose.py
@@ -65,10 +65,10 @@ class ComposeElement(Element):
# We name this variable 'integration' only to avoid
# collision with the Element.integrate() method.
- self.integration = self.node_get_member(node, bool, 'integrate')
- self.include = self.node_get_member(node, list, 'include')
- self.exclude = self.node_get_member(node, list, 'exclude')
- self.include_orphans = self.node_get_member(node, bool, 'include-orphans')
+ self.integration = node.get_bool('integrate')
+ self.include = node.get_sequence('include').as_str_list()
+ self.exclude = node.get_sequence('exclude').as_str_list()
+ self.include_orphans = node.get_bool('include-orphans')
def preflight(self):
pass
diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py
index 45847e685..bba56f351 100644
--- a/src/buildstream/plugins/elements/filter.py
+++ b/src/buildstream/plugins/elements/filter.py
@@ -171,9 +171,9 @@ class FilterElement(Element):
'include', 'exclude', 'include-orphans'
])
- self.include = self.node_get_member(node, list, 'include')
- self.exclude = self.node_get_member(node, list, 'exclude')
- self.include_orphans = self.node_get_member(node, bool, 'include-orphans')
+ self.include = node.get_sequence('include').as_str_list()
+ self.exclude = node.get_sequence('exclude').as_str_list()
+ self.include_orphans = node.get_bool('include-orphans')
self.include_provenance = self.node_provenance(node, member_name='include')
self.exclude_provenance = self.node_provenance(node, member_name='exclude')
@@ -217,7 +217,7 @@ class FilterElement(Element):
for dep in self.dependencies(Scope.BUILD, recurse=False):
# Check that all the included/excluded domains exist
pub_data = dep.get_public_data('bst')
- split_rules = self.node_get_member(pub_data, dict, 'split-rules', {})
+ split_rules = pub_data.get_mapping('split-rules', {})
unfound_includes = []
for domain in self.include:
if domain not in split_rules:
diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py
index 4222de360..d30e34828 100644
--- a/src/buildstream/plugins/elements/junction.py
+++ b/src/buildstream/plugins/elements/junction.py
@@ -175,9 +175,9 @@ class JunctionElement(Element):
BST_FORBID_RDEPENDS = True
def configure(self, node):
- self.path = self.node_get_member(node, str, 'path', default='')
- self.options = self.node_get_member(node, dict, 'options', default={})
- self.target = self.node_get_member(node, str, 'target', default=None)
+ self.path = node.get_str('path', default='')
+ self.options = node.get_mapping('options', default={})
+ self.target = node.get_str('target', default=None)
self.target_element = None
self.target_junction = None
diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py
index 0d194dcc1..1c694060c 100644
--- a/src/buildstream/plugins/elements/script.py
+++ b/src/buildstream/plugins/elements/script.py
@@ -46,7 +46,7 @@ class ScriptElement(buildstream.ScriptElement):
BST_VIRTUAL_DIRECTORY = True
def configure(self, node):
- for n in self.node_get_member(node, list, 'layout', []):
+ for n in node.get_sequence('layout', []):
dst = self.node_subst_member(n, 'destination')
elm = self.node_subst_member(n, 'element', None)
self.layout_add(elm, dst)
@@ -60,8 +60,7 @@ class ScriptElement(buildstream.ScriptElement):
self.set_work_dir()
self.set_install_root()
- self.set_root_read_only(self.node_get_member(node, bool,
- 'root-read-only', False))
+ self.set_root_read_only(node.get_bool('root-read-only', default=False))
# Plugin entry point
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
index b9b15e268..7b1785979 100644
--- a/src/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -77,8 +77,8 @@ class DownloadableFileSource(Source):
__urlopener = None
def configure(self, node):
- self.original_url = self.node_get_member(node, str, 'url')
- self.ref = self.node_get_member(node, str, 'ref', None)
+ self.original_url = node.get_str('url')
+ self.ref = node.get_str('ref', None)
self.url = self.translate_url(self.original_url)
self._warn_deprecated_etag(node)
@@ -99,7 +99,7 @@ class DownloadableFileSource(Source):
return Consistency.RESOLVED
def load_ref(self, node):
- self.ref = self.node_get_member(node, str, 'ref', None)
+ self.ref = node.get_str('ref', None)
self._warn_deprecated_etag(node)
def get_ref(self):
@@ -143,7 +143,7 @@ class DownloadableFileSource(Source):
.format(self.url, sha256, self.ref))
def _warn_deprecated_etag(self, node):
- etag = self.node_get_member(node, str, 'etag', None)
+ etag = node.get_str('etag', None)
if etag:
provenance = self.node_provenance(node, member_name='etag')
self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py
index e59986da6..28bea1415 100644
--- a/src/buildstream/plugins/sources/bzr.py
+++ b/src/buildstream/plugins/sources/bzr.py
@@ -69,9 +69,9 @@ class BzrSource(Source):
def configure(self, node):
self.node_validate(node, ['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS])
- self.original_url = self.node_get_member(node, str, 'url')
- self.tracking = self.node_get_member(node, str, 'track')
- self.ref = self.node_get_member(node, str, 'ref', None)
+ self.original_url = node.get_str('url')
+ self.tracking = node.get_str('track')
+ self.ref = node.get_str('ref', None)
self.url = self.translate_url(self.original_url)
def preflight(self):
@@ -93,7 +93,7 @@ class BzrSource(Source):
return Consistency.RESOLVED
def load_ref(self, node):
- self.ref = self.node_get_member(node, str, 'ref', None)
+ self.ref = node.get_str('ref', None)
def get_ref(self):
return self.ref
diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py
index e45994951..cc88cf53c 100644
--- a/src/buildstream/plugins/sources/deb.py
+++ b/src/buildstream/plugins/sources/deb.py
@@ -61,7 +61,7 @@ class DebSource(TarSource):
def configure(self, node):
super().configure(node)
- self.base_dir = self.node_get_member(node, str, 'base-dir', None)
+ self.base_dir = node.get_str('base-dir', None)
def preflight(self):
return
diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py
index e42868264..01117db78 100644
--- a/src/buildstream/plugins/sources/patch.py
+++ b/src/buildstream/plugins/sources/patch.py
@@ -57,7 +57,7 @@ class PatchSource(Source):
def configure(self, node):
self.path = self.node_get_project_path(node, 'path',
check_is_file=True)
- self.strip_level = self.node_get_member(node, int, "strip-level", 1)
+ self.strip_level = node.get_int("strip-level", default=1)
self.fullpath = os.path.join(self.get_project_directory(), self.path)
def preflight(self):
diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py
index 9d6c40d74..627f26397 100644
--- a/src/buildstream/plugins/sources/pip.py
+++ b/src/buildstream/plugins/sources/pip.py
@@ -111,17 +111,17 @@ class PipSource(Source):
def configure(self, node):
self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] +
Source.COMMON_CONFIG_KEYS)
- self.ref = self.node_get_member(node, str, 'ref', None)
- self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL)
+ self.ref = node.get_str('ref', None)
+ self.original_url = node.get_str('url', _PYPI_INDEX_URL)
self.index_url = self.translate_url(self.original_url)
- self.packages = self.node_get_member(node, list, 'packages', [])
- self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])
+ self.packages = node.get_sequence('packages', []).as_str_list()
+ self.requirements_files = node.get_sequence('requirements-files', []).as_str_list()
if not (self.packages or self.requirements_files):
raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))
def preflight(self):
- # Try to find a pip version that supports download command
+ # Try to find a pip version that spports download command
self.host_pip = None
for python in reversed(_PYTHON_VERSIONS):
try:
@@ -150,7 +150,7 @@ class PipSource(Source):
return self.ref
def load_ref(self, node):
- self.ref = self.node_get_member(node, str, 'ref', None)
+ self.ref = node.get_str('ref', None)
def set_ref(self, ref, node):
node['ref'] = self.ref = ref
diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py
index 562a8f226..bb4c11325 100644
--- a/src/buildstream/plugins/sources/remote.py
+++ b/src/buildstream/plugins/sources/remote.py
@@ -62,8 +62,8 @@ class RemoteSource(DownloadableFileSource):
def configure(self, node):
super().configure(node)
- self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
- self.executable = self.node_get_member(node, bool, 'executable', False)
+ self.filename = node.get_str('filename', os.path.basename(self.url))
+ self.executable = node.get_bool('executable', default=False)
if os.sep in self.filename:
raise SourceError('{}: filename parameter cannot contain directories'.format(self),
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
index c90de74ea..ef2ee1d85 100644
--- a/src/buildstream/plugins/sources/tar.py
+++ b/src/buildstream/plugins/sources/tar.py
@@ -72,8 +72,7 @@ class TarSource(DownloadableFileSource):
def configure(self, node):
super().configure(node)
- self.base_dir = self.node_get_member(node, str, 'base-dir', '*')
-
+ self.base_dir = node.get_str('base-dir', '*')
self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
def preflight(self):
diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py
index 9981f1260..1724f0587 100644
--- a/src/buildstream/plugins/sources/zip.py
+++ b/src/buildstream/plugins/sources/zip.py
@@ -72,8 +72,7 @@ class ZipSource(DownloadableFileSource):
def configure(self, node):
super().configure(node)
- self.base_dir = self.node_get_member(node, str, 'base-dir', '*')
-
+ self.base_dir = node.get_str('base-dir', '*')
self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
def get_unique_key(self):
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index d90b164bc..37ba8506b 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -112,7 +112,7 @@ class SandboxRemote(Sandbox):
def specs_from_config_node(config_node, basedir=None):
def require_node(config, keyname):
- val = _yaml.node_get(config, dict, keyname, default_value=None)
+ val = config.get_mapping(keyname, default=None)
if val is None:
provenance = _yaml.node_get_provenance(remote_config, key=keyname)
raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
@@ -121,7 +121,7 @@ class SandboxRemote(Sandbox):
.format(str(provenance), keyname))
return val
- remote_config = _yaml.node_get(config_node, dict, 'remote-execution', default_value=None)
+ remote_config = config_node.get_mapping('remote-execution', default=None)
if remote_config is None:
return None
@@ -131,7 +131,7 @@ class SandboxRemote(Sandbox):
exec_config = require_node(remote_config, 'execution-service')
storage_config = require_node(remote_config, 'storage-service')
- action_config = _yaml.node_get(remote_config, dict, 'action-cache-service', default_value={})
+ action_config = remote_config.get_mapping('action-cache-service', default={})
tls_keys = ['client-key', 'client-cert', 'server-cert']
@@ -173,7 +173,7 @@ class SandboxRemote(Sandbox):
for tls_key in tls_keys:
if tls_key in config:
- _yaml.node_set(config, tls_key, resolve_path(_yaml.node_get(config, str, tls_key)))
+ _yaml.node_set(config, tls_key, resolve_path(config.get_str(tls_key)))
return RemoteExecutionSpec(*[_yaml.node_sanitize(conf) for conf in service_configs])
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index 9fc9cf17d..609836551 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -258,8 +258,7 @@ class Source(Plugin):
All Sources derive from this class, this interface defines how
the core will be interacting with Sources.
"""
- __defaults = {} # The defaults from the project
- __defaults_set = False # Flag, in case there are not defaults at all
+ __defaults = None # The defaults from the project
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
"""Whether access to previous sources is required during track
@@ -1274,20 +1273,19 @@ class Source(Plugin):
@classmethod
def __init_defaults(cls, project, meta):
- if not cls.__defaults_set:
+ if cls.__defaults is None:
if meta.first_pass:
sources = project.first_pass_config.source_overrides
else:
sources = project.source_overrides
- cls.__defaults = _yaml.node_get(sources, dict, meta.kind, default_value={})
- cls.__defaults_set = True
+ cls.__defaults = sources.get_mapping(meta.kind, default={})
# This will resolve the final configuration to be handed
# off to source.configure()
#
@classmethod
def __extract_config(cls, meta):
- config = _yaml.node_get(cls.__defaults, dict, 'config', default_value={})
+ config = cls.__defaults.get_mapping('config', default={})
config = _yaml.node_copy(config)
_yaml.composite(config, meta.config)
diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py
index d682bb2ef..afbadbef7 100644
--- a/src/buildstream/testing/_sourcetests/mirror.py
+++ b/src/buildstream/testing/_sourcetests/mirror.py
@@ -357,9 +357,9 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
# Tracking tries upstream first. Check the ref is from upstream.
new_element = _yaml.load(element_path)
- source = _yaml.node_get(new_element, dict, 'sources', [0])
+ source = new_element.get_sequence('sources').mapping_at(0)
if 'ref' in source:
- assert _yaml.node_get(source, str, 'ref') == upstream_ref
+ assert source.get_str('ref') == upstream_ref
@pytest.mark.datafiles(DATA_DIR)
@@ -422,6 +422,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
# Check that tracking fell back to the mirror
new_element = _yaml.load(element_path)
- source = _yaml.node_get(new_element, dict, 'sources', [0])
+ source = new_element.get_sequence('sources').mapping_at(0)
if 'ref' in source:
- assert _yaml.node_get(source, str, 'ref') == mirror_ref
+ assert source.get_str('ref') == mirror_ref
diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py
index 668ea29e5..58679e9f1 100644
--- a/src/buildstream/testing/_sourcetests/track.py
+++ b/src/buildstream/testing/_sourcetests/track.py
@@ -321,13 +321,13 @@ def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
# Get all of the sources
assert 'sources' in new_sources
- sources_list = _yaml.node_get(new_sources, list, 'sources')
+ sources_list = new_sources.get_sequence('sources')
assert len(sources_list) == 1
# Get the first source from the sources list
- new_source = _yaml.node_get(new_sources, dict, 'sources', indices=[0])
+ new_source = sources_list.mapping_at(0)
assert 'ref' in new_source
- assert ref == _yaml.node_get(new_source, str, 'ref')
+ assert ref == new_source.get_str('ref')
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index d89c834e0..f9c80afff 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -239,8 +239,8 @@ def test_filter_track(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0])
- new_input_ref = _yaml.node_get(source_node, str, 'ref')
+ source_node = new_input.get_sequence('sources').mapping_at(0)
+ new_input_ref = source_node.get_str('ref')
assert new_input_ref == ref
@@ -294,7 +294,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0])
+ source_node = new_input.get_sequence('sources').mapping_at(0)
assert 'ref' not in source_node
@@ -348,8 +348,8 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0])
- new_ref = _yaml.node_get(source_node, str, 'ref')
+ source_node = new_input.get_sequence('sources').mapping_at(0)
+ new_ref = source_node.get_str('ref')
assert new_ref == ref
@@ -413,13 +413,13 @@ def test_filter_track_multi(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0])
- new_ref = _yaml.node_get(source_node, str, 'ref')
+ source_node = new_input.get_sequence('sources').mapping_at(0)
+ new_ref = source_node.get_str('ref')
assert new_ref == ref
new_input2 = _yaml.load(input2_file)
- source_node2 = _yaml.node_get(new_input2, dict, 'sources', indices=[0])
- new_ref2 = _yaml.node_get(source_node2, str, 'ref')
+ source_node2 = new_input2.get_sequence('sources').mapping_at(0)
+ new_ref2 = source_node2.get_str('ref')
assert new_ref2 == ref
@@ -482,12 +482,12 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
# Now check that a ref field exists
new_input = _yaml.load(input_file)
- source_node = _yaml.node_get(new_input, dict, 'sources', indices=[0])
+ source_node = new_input.get_sequence('sources').mapping_at(0)
assert 'ref' not in source_node
new_input2 = _yaml.load(input2_file)
- source_node2 = _yaml.node_get(new_input2, dict, 'sources', indices=[0])
- new_ref2 = _yaml.node_get(source_node2, str, 'ref')
+ source_node2 = new_input2.get_sequence('sources').mapping_at(0)
+ new_ref2 = source_node2.get_str('ref')
assert new_ref2 == ref
diff --git a/tests/elements/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py
index c6d128b72..b4a556244 100644
--- a/tests/elements/filter/basic/element_plugins/dynamic.py
+++ b/tests/elements/filter/basic/element_plugins/dynamic.py
@@ -5,7 +5,7 @@ from buildstream import Element, Scope
class DynamicElement(Element):
def configure(self, node):
self.node_validate(node, ['split-rules'])
- self.split_rules = self.node_get_member(node, dict, 'split-rules')
+ self.split_rules = node.get_mapping('split-rules')
def preflight(self):
pass
diff --git a/tests/format/include.py b/tests/format/include.py
index bfadce7ed..e2e16c34a 100644
--- a/tests/format/include.py
+++ b/tests/format/include.py
@@ -28,7 +28,7 @@ def test_include_project_file(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, bool, 'included')
+ assert loaded.get_bool('included')
def test_include_missing_file(cli, tmpdir):
@@ -87,7 +87,7 @@ def test_include_junction_file(cli, tmpdir, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, bool, 'included')
+ assert loaded.get_bool('included')
@pytest.mark.datafiles(DATA_DIR)
@@ -102,7 +102,7 @@ def test_include_junction_options(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'build_arch') == 'x86_64'
+ assert loaded.get_str('build_arch') == 'x86_64'
@pytest.mark.datafiles(DATA_DIR)
@@ -135,7 +135,7 @@ def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
'junction.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'included', default_value=None) is None
+ assert loaded.get_str('included', default=None) is None
@pytest.mark.datafiles(DATA_DIR)
@@ -168,7 +168,7 @@ def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles):
'junction.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'included', default_value=None) is not None
+ assert loaded.get_str('included', default=None) is not None
@pytest.mark.datafiles(DATA_DIR)
@@ -182,8 +182,8 @@ def test_include_element_overrides(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'manual_main_override', default_value=None) is not None
- assert _yaml.node_get(loaded, str, 'manual_included_override', default_value=None) is not None
+ assert loaded.get_str('manual_main_override', default=None) is not None
+ assert loaded.get_str('manual_included_override', default=None) is not None
@pytest.mark.datafiles(DATA_DIR)
@@ -197,7 +197,7 @@ def test_include_element_overrides_composition(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, list, 'build-commands') == ['first', 'second']
+ assert loaded.get_sequence('build-commands').as_str_list() == ['first', 'second']
@pytest.mark.datafiles(DATA_DIR)
@@ -213,9 +213,9 @@ def test_list_overide_does_not_fail_upon_first_composition(cli, datafiles):
loaded = _yaml.load_data(result.output)
# Assert that the explicitly overwritten public data is present
- bst = _yaml.node_get(loaded, dict, 'bst')
+ bst = loaded.get_mapping('bst')
assert 'foo-commands' in bst
- assert _yaml.node_get(bst, list, 'foo-commands') == ['need', 'this']
+ assert bst.get_sequence('foo-commands').as_str_list() == ['need', 'this']
@pytest.mark.datafiles(DATA_DIR)
@@ -229,7 +229,7 @@ def test_include_element_overrides_sub_include(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'included', default_value=None) is not None
+ assert loaded.get_str('included', default=None) is not None
@pytest.mark.datafiles(DATA_DIR)
@@ -248,8 +248,8 @@ def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
'junction.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'main_override', default_value=None) is not None
- assert _yaml.node_get(loaded, str, 'included_override', default_value=None) is None
+ assert loaded.get_str('main_override', default=None) is not None
+ assert loaded.get_str('included_override', default=None) is None
@pytest.mark.datafiles(DATA_DIR)
@@ -264,7 +264,7 @@ def test_conditional_in_fragment(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'size') == '8'
+ assert loaded.get_str('size') == '8'
@pytest.mark.datafiles(DATA_DIR)
@@ -278,7 +278,7 @@ def test_inner(cli, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'build_arch') == 'x86_64'
+ assert loaded.get_str('build_arch') == 'x86_64'
@pytest.mark.datafiles(DATA_DIR)
@@ -310,4 +310,4 @@ def test_local_to_junction(cli, tmpdir, datafiles):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, bool, 'included')
+ assert loaded.get_bool('included')
diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py
index 4afde817d..b922c7a0e 100644
--- a/tests/format/include_composition.py
+++ b/tests/format/include_composition.py
@@ -14,7 +14,7 @@ def make_includes(basedir):
return Includes(loader)
-def test_main_has_prority(tmpdir):
+def test_main_has_priority(tmpdir):
includes = make_includes(str(tmpdir))
_yaml.dump({'(@)': ['a.yml'],
@@ -28,7 +28,7 @@ def test_main_has_prority(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['main']
+ assert main.get_sequence('test').as_str_list() == ['main']
def test_include_cannot_append(tmpdir):
@@ -44,7 +44,7 @@ def test_include_cannot_append(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['main']
+ assert main.get_sequence('test').as_str_list() == ['main']
def test_main_can_append(tmpdir):
@@ -60,7 +60,7 @@ def test_main_can_append(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['a', 'main']
+ assert main.get_sequence('test').as_str_list() == ['a', 'main']
def test_sibling_cannot_append_backward(tmpdir):
@@ -77,7 +77,7 @@ def test_sibling_cannot_append_backward(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['b']
+ assert main.get_sequence('test').as_str_list() == ['b']
def test_sibling_can_append_forward(tmpdir):
@@ -94,7 +94,7 @@ def test_sibling_can_append_forward(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['a', 'b']
+ assert main.get_sequence('test').as_str_list() == ['a', 'b']
def test_lastest_sibling_has_priority(tmpdir):
@@ -111,7 +111,7 @@ def test_lastest_sibling_has_priority(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['b']
+ assert main.get_sequence('test').as_str_list() == ['b']
def test_main_keeps_keys(tmpdir):
@@ -127,8 +127,8 @@ def test_main_keeps_keys(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['a']
- assert _yaml.node_get(main, str, 'something') == 'else'
+ assert main.get_sequence('test').as_str_list() == ['a']
+ assert main.get_str('something') == 'else'
def test_overwrite_directive_on_later_composite(tmpdir):
@@ -151,5 +151,5 @@ def test_overwrite_directive_on_later_composite(tmpdir):
includes.process(main)
- assert _yaml.node_get(main, list, 'test') == ['Overwritten']
- assert _yaml.node_get(main, str, 'foo') == 'should be present'
+ assert main.get_sequence('test').as_str_list() == ['Overwritten']
+ assert main.get_str('foo') == 'should be present'
diff --git a/tests/format/optionarch.py b/tests/format/optionarch.py
index a35ac685d..f347e27ae 100644
--- a/tests/format/optionarch.py
+++ b/tests/format/optionarch.py
@@ -48,7 +48,7 @@ def test_conditional(cli, datafiles, machine, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'result') == expected
+ assert loaded.get_str('result') == expected
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/format/optionbool.py b/tests/format/optionbool.py
index 0d1ee601e..d772b483c 100644
--- a/tests/format/optionbool.py
+++ b/tests/format/optionbool.py
@@ -42,7 +42,7 @@ def test_conditional_cli(cli, datafiles, target, option, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'thepony') == expected
+ assert loaded.get_str('thepony') == expected
# Test configuration of boolean option in the config file
@@ -71,7 +71,7 @@ def test_conditional_config(cli, datafiles, target, option, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'thepony') == expected
+ assert loaded.get_str('thepony') == expected
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/format/optioneltmask.py b/tests/format/optioneltmask.py
index d33b5771c..75265fdd7 100644
--- a/tests/format/optioneltmask.py
+++ b/tests/format/optioneltmask.py
@@ -28,7 +28,7 @@ def test_conditional_cli(cli, datafiles, target, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'debug') == expected
+ assert loaded.get_str('debug') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -56,7 +56,7 @@ def test_conditional_config(cli, datafiles, target, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'debug') == expected
+ assert loaded.get_str('debug') == expected
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/format/optionenum.py b/tests/format/optionenum.py
index b8a96b0c2..f9aff503f 100644
--- a/tests/format/optionenum.py
+++ b/tests/format/optionenum.py
@@ -33,7 +33,7 @@ def test_conditional_cli(cli, datafiles, target, option, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'result') == expected
+ assert loaded.get_str('result') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -66,7 +66,7 @@ def test_conditional_config(cli, datafiles, target, option, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'result') == expected
+ assert loaded.get_str('result') == expected
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/format/optionexports.py b/tests/format/optionexports.py
index 5df7522b5..104abcf83 100644
--- a/tests/format/optionexports.py
+++ b/tests/format/optionexports.py
@@ -36,4 +36,4 @@ def test_export(cli, datafiles, option_name, option_value, var_name, var_value):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, var_name) == var_value
+ assert loaded.get_str(var_name) == var_value
diff --git a/tests/format/optionflags.py b/tests/format/optionflags.py
index e28c54236..29bb7ec2c 100644
--- a/tests/format/optionflags.py
+++ b/tests/format/optionflags.py
@@ -39,7 +39,7 @@ def test_conditional_cli(cli, datafiles, target, option, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'result') == expected
+ assert loaded.get_str('result') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -69,7 +69,7 @@ def test_conditional_config(cli, datafiles, target, option, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'result') == expected
+ assert loaded.get_str('result') == expected
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/format/optionos.py b/tests/format/optionos.py
index 57277b106..f915d889e 100644
--- a/tests/format/optionos.py
+++ b/tests/format/optionos.py
@@ -47,7 +47,7 @@ def test_conditionals(cli, datafiles, system, value, expected):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'result') == expected
+ assert loaded.get_str('result') == expected
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/format/optionoverrides.py b/tests/format/optionoverrides.py
index 60d02b3a3..d4ed257dd 100644
--- a/tests/format/optionoverrides.py
+++ b/tests/format/optionoverrides.py
@@ -29,4 +29,4 @@ def test_override(cli, datafiles, arch):
expected_value = '--host={}-unknown-linux-gnu'.format(arch)
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'conf-global') == expected_value
+ assert loaded.get_str('conf-global') == expected_value
diff --git a/tests/format/options.py b/tests/format/options.py
index 3a8210dc3..9c0e043f9 100644
--- a/tests/format/options.py
+++ b/tests/format/options.py
@@ -136,7 +136,7 @@ def test_simple_conditional(cli, datafiles, opt_option, expected_prefix):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'prefix') == expected_prefix
+ assert loaded.get_str('prefix') == expected_prefix
@pytest.mark.datafiles(DATA_DIR)
@@ -159,7 +159,7 @@ def test_nested_conditional(cli, datafiles, debug, logging, expected):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'debug') == expected
+ assert loaded.get_str('debug') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -182,7 +182,7 @@ def test_compound_and_conditional(cli, datafiles, debug, logging, expected):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'debug') == expected
+ assert loaded.get_str('debug') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -205,7 +205,7 @@ def test_compound_or_conditional(cli, datafiles, debug, logging, expected):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert _yaml.node_get(loaded, str, 'logging') == expected
+ assert loaded.get_str('logging') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -223,10 +223,10 @@ def test_deep_nesting_level1(cli, datafiles, option, expected):
'element.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- shallow_list = _yaml.node_get(loaded, list, 'shallow-nest')
- first_dict = shallow_list[0]
+ shallow_list = loaded.get_sequence('shallow-nest')
+ first_dict = shallow_list.mapping_at(0)
- assert _yaml.node_get(first_dict, str, 'animal') == expected
+ assert first_dict.get_str('animal') == expected
@pytest.mark.datafiles(DATA_DIR)
@@ -244,8 +244,8 @@ def test_deep_nesting_level2(cli, datafiles, option, expected):
'element-deeper.bst'])
result.assert_success()
loaded = _yaml.load_data(result.output)
- shallow_list = _yaml.node_get(loaded, list, 'deep-nest')
- deeper_list = shallow_list[0]
- first_dict = deeper_list[0]
+ shallow_list = loaded.get_sequence('deep-nest')
+ deeper_list = shallow_list.sequence_at(0)
+ first_dict = deeper_list.mapping_at(0)
- assert _yaml.node_get(first_dict, str, 'animal') == expected
+ assert first_dict.get_str('animal') == expected
diff --git a/tests/format/project.py b/tests/format/project.py
index fbb742d47..697fe7e3d 100644
--- a/tests/format/project.py
+++ b/tests/format/project.py
@@ -82,8 +82,8 @@ def test_load_default_project(cli, datafiles):
# Read back some of our project defaults from the env
env = _yaml.load_data(result.output)
- assert _yaml.node_get(env, str, 'USER') == "tomjon"
- assert _yaml.node_get(env, str, 'TERM') == "dumb"
+ assert env.get_str('USER') == "tomjon"
+ assert env.get_str('TERM') == "dumb"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@@ -97,8 +97,8 @@ def test_load_project_from_subdir(cli, datafiles):
# Read back some of our project defaults from the env
env = _yaml.load_data(result.output)
- assert _yaml.node_get(env, str, 'USER') == "tomjon"
- assert _yaml.node_get(env, str, 'TERM') == "dumb"
+ assert env.get_str('USER') == "tomjon"
+ assert env.get_str('TERM') == "dumb"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@@ -111,7 +111,7 @@ def test_override_project_path(cli, datafiles):
# Read back the overridden path
env = _yaml.load_data(result.output)
- assert _yaml.node_get(env, str, 'PATH') == "/bin:/sbin"
+ assert env.get_str('PATH') == "/bin:/sbin"
@pytest.mark.datafiles(os.path.join(DATA_DIR))
diff --git a/tests/format/projectoverrides.py b/tests/format/projectoverrides.py
index 4b0c3f4d0..730e43b1e 100644
--- a/tests/format/projectoverrides.py
+++ b/tests/format/projectoverrides.py
@@ -24,6 +24,6 @@ def test_prepend_configure_commands(cli, datafiles):
result.assert_success()
loaded = _yaml.load_data(result.output)
- config_commands = _yaml.node_get(loaded, list, 'configure-commands')
+ config_commands = loaded.get_sequence('configure-commands').as_str_list()
assert len(config_commands) == 3
assert config_commands[0] == 'echo "Hello World!"'
diff --git a/tests/format/variables.py b/tests/format/variables.py
index 87e3b2903..ed5407b83 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -49,7 +49,7 @@ def test_defaults(cli, datafiles, target, varname, expected):
])
result.assert_success()
result_vars = _yaml.load_data(result.output)
- assert _yaml.node_get(result_vars, str, varname) == expected
+ assert result_vars.get_str(varname) == expected
################################################################
@@ -75,7 +75,7 @@ def test_overrides(cli, datafiles, target, varname, expected):
])
result.assert_success()
result_vars = _yaml.load_data(result.output)
- assert _yaml.node_get(result_vars, str, varname) == expected
+ assert result_vars.get_str(varname) == expected
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing_variables'))
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 97bce91a7..f9cd1cba4 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -15,7 +15,7 @@ from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream import utils
-from tests.testutils import generate_junction, yaml_file_get_provenance, create_artifact_share
+from tests.testutils import generate_junction, create_artifact_share
from . import configure_project
@@ -460,8 +460,9 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- provenance = yaml_file_get_provenance(
- element_path, 'junction-dep.bst', key='depends', indices=[0])
+ element_node = _yaml.load(element_path, shortname='junction-dep.bst')
+ ref_node = element_node.get_sequence('depends').mapping_at(0)
+ provenance = _yaml.node_get_provenance(ref_node)
assert str(provenance) in result.stderr
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index 81fd43487..38aafa558 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -75,11 +75,12 @@ def test_list_cross_junction(cli, tmpdir):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list)
- workspaces = _yaml.node_get(loaded, list, 'workspaces')
+ workspaces = loaded.get_sequence('workspaces')
assert len(workspaces) == 1
- assert 'element' in workspaces[0]
- assert _yaml.node_get(workspaces[0], str, 'element') == element
+ first_workspace = workspaces.mapping_at(0)
+
+ assert 'element' in first_workspace
+ assert first_workspace.get_str('element') == element
def test_close_cross_junction(cli, tmpdir):
@@ -97,8 +98,7 @@ def test_close_cross_junction(cli, tmpdir):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list)
- workspaces = _yaml.node_get(loaded, list, 'workspaces')
+ workspaces = loaded.get_sequence('workspaces')
assert not workspaces
@@ -116,8 +116,7 @@ def test_close_all_cross_junction(cli, tmpdir):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list)
- workspaces = _yaml.node_get(loaded, list, 'workspaces')
+ workspaces = loaded.get_sequence('workspaces')
assert not workspaces
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index f470ce2b7..d7e0ecafb 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -9,7 +9,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import generate_junction, yaml_file_get_provenance
+from tests.testutils import generate_junction
from . import configure_project
@@ -171,6 +171,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- provenance = yaml_file_get_provenance(
- element_path, 'junction-dep.bst', key='depends', indices=[0])
+ element_node = _yaml.load(element_path, shortname='junction-dep.bst')
+ ref_node = element_node.get_sequence('depends').mapping_at(0)
+ provenance = _yaml.node_get_provenance(ref_node)
assert str(provenance) in result.stderr
diff --git a/tests/frontend/init.py b/tests/frontend/init.py
index 0eac5d528..e9d92be90 100644
--- a/tests/frontend/init.py
+++ b/tests/frontend/init.py
@@ -19,9 +19,9 @@ def test_defaults(cli, tmpdir):
result.assert_success()
project_conf = _yaml.load(project_path)
- assert _yaml.node_get(project_conf, str, 'name') == 'foo'
- assert _yaml.node_get(project_conf, str, 'format-version') == str(BST_FORMAT_VERSION)
- assert _yaml.node_get(project_conf, str, 'element-path') == 'elements'
+ assert project_conf.get_str('name') == 'foo'
+ assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION)
+ assert project_conf.get_str('element-path') == 'elements'
def test_all_options(cli, tmpdir):
@@ -37,9 +37,9 @@ def test_all_options(cli, tmpdir):
result.assert_success()
project_conf = _yaml.load(project_path)
- assert _yaml.node_get(project_conf, str, 'name') == 'foo'
- assert _yaml.node_get(project_conf, str, 'format-version') == str(2)
- assert _yaml.node_get(project_conf, str, 'element-path') == 'ponies'
+ assert project_conf.get_str('name') == 'foo'
+ assert project_conf.get_str('format-version') == str(2)
+ assert project_conf.get_str('element-path') == 'ponies'
elements_dir = os.path.join(project, 'ponies')
assert os.path.isdir(elements_dir)
@@ -70,8 +70,8 @@ def test_force_overwrite_project(cli, tmpdir):
result.assert_success()
project_conf = _yaml.load(project_path)
- assert _yaml.node_get(project_conf, str, 'name') == 'foo'
- assert _yaml.node_get(project_conf, str, 'format-version') == str(BST_FORMAT_VERSION)
+ assert project_conf.get_str('name') == 'foo'
+ assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION)
@pytest.mark.parametrize("project_name", [('Micheal Jackson'), ('one+one')])
@@ -122,6 +122,6 @@ def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path):
assert full_element_path.exists()
project_conf = _yaml.load(str(project_conf_path))
- assert _yaml.node_get(project_conf, str, 'name') == 'project_name'
- assert _yaml.node_get(project_conf, str, 'format-version') == '0'
- assert _yaml.node_get(project_conf, str, 'element-path') == element_path
+ assert project_conf.get_str('name') == 'project_name'
+ assert project_conf.get_str('format-version') == '0'
+ assert project_conf.get_str('element-path') == element_path
diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py
index f17c14029..7c0df4595 100644
--- a/tests/frontend/project/sources/fetch_source.py
+++ b/tests/frontend/project/sources/fetch_source.py
@@ -38,11 +38,11 @@ class FetchFetcher(SourceFetcher):
class FetchSource(Source):
# Read config to know which URLs to fetch
def configure(self, node):
- self.original_urls = self.node_get_member(node, list, 'urls')
- self.output_file = self.node_get_member(node, str, 'output-text')
+ self.original_urls = node.get_sequence('urls').as_str_list()
+ self.output_file = node.get_str('output-text')
self.fetch_succeeds = {}
if 'fetch-succeeds' in node:
- fetch_succeeds_node = self.node_get_member(node, dict, 'fetch-succeeds')
+ fetch_succeeds_node = node.get_mapping('fetch-succeeds')
for key, value in self.node_items(fetch_succeeds_node):
self.fetch_succeeds[key] = value in ('True', 'true')
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index 4ef97dd84..4192ba1e5 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -10,7 +10,7 @@ from buildstream.testing import cli # pylint: disable=unused-import
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import generate_junction, yaml_file_get_provenance
+from tests.testutils import generate_junction
from . import configure_project
@@ -333,8 +333,9 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced):
etc_result.assert_success()
else:
# Assert that we have the expected provenance encoded into the error
- provenance = yaml_file_get_provenance(
- element_path, 'junction-dep.bst', key='depends', indices=[0])
+ element_node = _yaml.load(element_path, shortname='junction-dep.bst')
+ ref_node = element_node.get_sequence('depends').mapping_at(0)
+ provenance = _yaml.node_get_provenance(ref_node)
assert str(provenance) in dep_result.stderr
dep_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 808bf0593..1163cc687 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -9,7 +9,7 @@ from buildstream.testing import create_repo
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream import _yaml
-from tests.testutils import generate_junction, yaml_file_get_provenance
+from tests.testutils import generate_junction
from . import configure_project
# Project directory
@@ -275,8 +275,9 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage):
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- provenance = yaml_file_get_provenance(
- element_path, 'junction-dep.bst', key='depends', indices=[0])
+ element_node = _yaml.load(element_path, shortname='junction-dep.bst')
+ ref_node = element_node.get_sequence('depends').mapping_at(0)
+ provenance = _yaml.node_get_provenance(ref_node)
assert str(provenance) in result.stderr
@@ -313,8 +314,9 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage):
result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
# Assert that we have the expected provenance encoded into the error
- provenance = yaml_file_get_provenance(
- element_path, 'junction-dep.bst', key='depends', indices=[0])
+ element_node = _yaml.load(element_path, shortname='junction-dep.bst')
+ ref_node = element_node.get_sequence('depends').mapping_at(0)
+ provenance = _yaml.node_get_provenance(ref_node)
assert str(provenance) in result.stderr
# Now track the junction itself
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index 5267f8e56..b240e5728 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -184,11 +184,11 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
# Check that the correct origin branch is set
element_config = _yaml.load(os.path.join(project, "elements", element_name))
- source_config = _yaml.node_get(element_config, dict, 'sources', [0])
+ source_config = element_config.get_sequence('sources').mapping_at(0)
output = subprocess.check_output(["bzr", "info"], cwd=workspace)
- stripped_url = _yaml.node_get(source_config, str, 'url').lstrip("file:///")
+ stripped_url = source_config.get_str('url').lstrip("file:///")
expected_output_str = ("checkout of branch: /{}/{}"
- .format(stripped_url, _yaml.node_get(source_config, str, 'track')))
+ .format(stripped_url, source_config.get_str('track')))
assert expected_output_str in str(output)
@@ -608,13 +608,12 @@ def test_list(cli, tmpdir, datafiles):
result.assert_success()
loaded = _yaml.load_data(result.output)
- assert isinstance(_yaml.node_get(loaded, None, 'workspaces'), list)
- workspaces = _yaml.node_get(loaded, list, 'workspaces')
+ workspaces = loaded.get_sequence('workspaces')
assert len(workspaces) == 1
- space = workspaces[0]
- assert _yaml.node_get(space, str, 'element') == element_name
- assert _yaml.node_get(space, str, 'directory') == workspace
+ space = workspaces.mapping_at(0)
+ assert space.get_str('element') == element_name
+ assert space.get_str('directory') == workspace
@pytest.mark.datafiles(DATA_DIR)
@@ -1133,7 +1132,7 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
# Delete the ref from the source so that we can detect if the
# element has been tracked
element_contents = _yaml.load(element_file)
- _yaml.node_del(_yaml.node_get(element_contents, dict, 'sources', [0]), 'ref')
+ _yaml.node_del(element_contents.get_sequence('sources').mapping_at(0), 'ref')
_yaml.dump(element_contents, element_file)
result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm])
@@ -1141,7 +1140,7 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
# Element is tracked now
element_contents = _yaml.load(element_file)
- assert 'ref' in _yaml.node_get(element_contents, dict, 'sources', [0])
+ assert 'ref' in element_contents.get_sequence('sources').mapping_at(0)
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/internals/yaml.py b/tests/internals/yaml.py
index cd3bdc535..aa83d949a 100644
--- a/tests/internals/yaml.py
+++ b/tests/internals/yaml.py
@@ -103,16 +103,16 @@ def test_node_get(datafiles):
base = _yaml.load(filename)
assert base.value.get('kind').value == 'pony'
- children = _yaml.node_get(base, list, 'children')
- assert isinstance(children, list)
+ children = base.get_sequence('children')
+ assert isinstance(children, _yaml.SequenceNode)
assert len(children) == 7
- child = _yaml.node_get(base, dict, 'children', indices=[6])
+ child = base.get_sequence('children').mapping_at(6)
assert_provenance(filename, 20, 8, child, 'mood')
- extra = _yaml.node_get(base, dict, 'extra')
+ extra = base.get_mapping('extra')
with pytest.raises(LoadError) as exc:
- _yaml.node_get(extra, dict, 'old')
+ extra.get_mapping('old')
assert exc.value.reason == LoadErrorReason.INVALID_DATA
@@ -128,7 +128,7 @@ def test_node_set(datafiles):
assert 'mother' not in base
_yaml.node_set(base, 'mother', 'snow white')
- assert _yaml.node_get(base, str, 'mother') == 'snow white'
+ assert base.get_str('mother') == 'snow white'
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@@ -141,14 +141,14 @@ def test_node_set_overwrite(datafiles):
base = _yaml.load(filename)
# Overwrite a string
- assert _yaml.node_get(base, str, 'kind') == 'pony'
+ assert base.get_str('kind') == 'pony'
_yaml.node_set(base, 'kind', 'cow')
- assert _yaml.node_get(base, str, 'kind') == 'cow'
+ assert base.get_str('kind') == 'cow'
# Overwrite a list as a string
- assert _yaml.node_get(base, list, 'moods') == ['happy', 'sad']
+ assert base.get_sequence('moods').as_str_list() == ['happy', 'sad']
_yaml.node_set(base, 'moods', 'unemotional')
- assert _yaml.node_get(base, str, 'moods') == 'unemotional'
+ assert base.get_str('moods') == 'unemotional'
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@@ -160,13 +160,11 @@ def test_node_set_list_element(datafiles):
base = _yaml.load(filename)
- assert _yaml.node_get(base, list, 'moods') == ['happy', 'sad']
- assert _yaml.node_get(base, str, 'moods', indices=[0]) == 'happy'
+ assert base.get_sequence('moods').as_str_list() == ['happy', 'sad']
_yaml.node_set(base, 'moods', 'confused', indices=[0])
- assert _yaml.node_get(base, list, 'moods') == ['confused', 'sad']
- assert _yaml.node_get(base, str, 'moods', indices=[0]) == 'confused'
+ assert base.get_sequence('moods').as_str_list() == ['confused', 'sad']
# Really this is testing _yaml.node_copy(), we want to
@@ -188,14 +186,14 @@ def test_composite_preserve_originals(datafiles):
base_copy = _yaml.node_copy(base)
_yaml.composite_dict(base_copy, overlay)
- copy_extra = _yaml.node_get(base_copy, dict, 'extra')
- orig_extra = _yaml.node_get(base, dict, 'extra')
+ copy_extra = base_copy.get_mapping('extra')
+ orig_extra = base.get_mapping('extra')
# Test that the node copy has the overridden value...
- assert _yaml.node_get(copy_extra, str, 'old') == 'override'
+ assert copy_extra.get_str('old') == 'override'
# But the original node is not effected by the override.
- assert _yaml.node_get(orig_extra, str, 'old') == 'new'
+ assert orig_extra.get_str('old') == 'new'
# Tests for list composition
@@ -254,11 +252,11 @@ def test_list_composition(datafiles, filename, tmpdir,
_yaml.composite_dict(base, overlay)
- children = _yaml.node_get(base, list, 'children')
+ children = base.get_sequence('children')
assert len(children) == length
- child = children[index]
+ child = children.mapping_at(index)
- assert _yaml.node_get(child, str, 'mood') == mood
+ assert child.get_str('mood') == mood
assert_provenance(prov_file, prov_line, prov_col, child, 'mood')
@@ -272,7 +270,7 @@ def test_list_deletion(datafiles):
overlay = _yaml.load(overlay, shortname='listoverwriteempty.yaml')
_yaml.composite_dict(base, overlay)
- children = _yaml.node_get(base, list, 'children')
+ children = base.get_sequence('children')
assert not children
@@ -286,8 +284,8 @@ def test_nonexistent_list_extension(datafiles):
_yaml.node_extend_list(base, 'todo', 3, 'empty')
- assert len(_yaml.node_get(base, list, 'todo')) == 3
- assert _yaml.node_get(base, list, 'todo') == ['empty', 'empty', 'empty']
+ assert len(base.get_sequence('todo')) == 3
+ assert base.get_sequence('todo').as_str_list() == ['empty', 'empty', 'empty']
# Tests for deep list composition
@@ -390,11 +388,11 @@ def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
_yaml.composite_dict(base, overlay1)
_yaml.composite_dict(base, overlay2)
- children = _yaml.node_get(base, list, 'children')
+ children = base.get_sequence('children')
assert len(children) == length
- child = children[index]
+ child = children.mapping_at(index)
- assert _yaml.node_get(child, str, 'mood') == mood
+ assert child.get_str('mood') == mood
assert_provenance(prov_file, prov_line, prov_col, child, 'mood')
#####################
@@ -407,11 +405,11 @@ def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
_yaml.composite_dict(overlay1, overlay2)
_yaml.composite_dict(base, overlay1)
- children = _yaml.node_get(base, list, 'children')
+ children = base.get_sequence('children')
assert len(children) == length
- child = children[index]
+ child = children.mapping_at(index)
- assert _yaml.node_get(child, str, 'mood') == mood
+ assert child.get_str('mood') == mood
assert_provenance(prov_file, prov_line, prov_col, child, 'mood')
@@ -424,19 +422,19 @@ def test_convert_value_to_string(datafiles):
# Run file through yaml to convert it
test_dict = _yaml.load(conf_file)
- user_config = _yaml.node_get(test_dict, str, "Test1")
+ user_config = test_dict.get_str("Test1")
assert isinstance(user_config, str)
assert user_config == "1_23_4"
- user_config = _yaml.node_get(test_dict, str, "Test2")
+ user_config = test_dict.get_str("Test2")
assert isinstance(user_config, str)
assert user_config == "1.23.4"
- user_config = _yaml.node_get(test_dict, str, "Test3")
+ user_config = test_dict.get_str("Test3")
assert isinstance(user_config, str)
assert user_config == "1.20"
- user_config = _yaml.node_get(test_dict, str, "Test4")
+ user_config = test_dict.get_str("Test4")
assert isinstance(user_config, str)
assert user_config == "OneTwoThree"
@@ -451,7 +449,7 @@ def test_value_doesnt_match_expected(datafiles):
test_dict = _yaml.load(conf_file)
with pytest.raises(LoadError) as exc:
- _yaml.node_get(test_dict, int, "Test4")
+ test_dict.get_int("Test4")
assert exc.value.reason == LoadErrorReason.INVALID_DATA
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 45a2b827a..69155e02b 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -760,7 +760,7 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
result.assert_success()
element = _yaml.load(element_path)
- new_ref = _yaml.node_get(_yaml.node_get(element, dict, 'sources', [0]), str, 'ref')
+ new_ref = element.get_sequence('sources').mapping_at(0).get_str('ref')
if ref_format == 'git-describe' and tag:
# Check and strip prefix
@@ -849,16 +849,16 @@ def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
if ref_storage == 'inline':
element = _yaml.load(element_path)
- tags = _yaml.node_get(_yaml.node_get(element, dict, 'sources', [0]), list, 'tags')
+ tags = element.get_sequence('sources').mapping_at(0).get_sequence('tags')
assert len(tags) == 2
for tag in tags:
assert 'tag' in tag
assert 'commit' in tag
assert 'annotated' in tag
- assert _yaml.node_get(tag, bool, 'annotated') == (tag_type == 'annotated')
+ assert tag.get_bool('annotated') == (tag_type == 'annotated')
- assert {(_yaml.node_get(tag, str, 'tag'),
- _yaml.node_get(tag, str, 'commit'))
+ assert {(tag.get_str('tag'),
+ tag.get_str('commit'))
for tag in tags} == {('tag1', repo.rev_parse('tag1^{commit}')),
('tag2', repo.rev_parse('tag2^{commit}'))}
@@ -962,18 +962,18 @@ def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_ty
if ref_storage == 'inline':
element = _yaml.load(element_path)
- source = _yaml.node_get(element, dict, 'sources', indices=[0])
- tags = _yaml.node_get(source, list, 'tags')
+ source = element.get_sequence('sources').mapping_at(0)
+ tags = source.get_sequence('tags')
assert len(tags) == 1
- tag = _yaml.node_get(source, dict, 'tags', indices=[0])
+ tag = source.get_sequence('tags').mapping_at(0)
assert 'tag' in tag
assert 'commit' in tag
assert 'annotated' in tag
- assert _yaml.node_get(tag, bool, 'annotated') == (tag_type == 'annotated')
+ assert tag.get_bool('annotated') == (tag_type == 'annotated')
- tag_name = _yaml.node_get(tag, str, 'tag')
- commit = _yaml.node_get(tag, str, 'commit')
+ tag_name = tag.get_str('tag')
+ commit = tag.get_str('commit')
assert (tag_name, commit) == ('tag', repo.rev_parse('tag^{commit}'))
checkout = os.path.join(str(tmpdir), 'checkout')
@@ -1125,7 +1125,7 @@ def test_default_do_not_track_tags(cli, tmpdir, datafiles):
result.assert_success()
element = _yaml.load(element_path)
- source = _yaml.node_get(element, dict, 'sources', indices=[0])
+ source = element.get_sequence('sources').mapping_at(0)
assert 'tags' not in source
diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py
index 800e0ced5..50bfe383c 100644
--- a/tests/sources/previous_source_access.py
+++ b/tests/sources/previous_source_access.py
@@ -24,7 +24,7 @@ def test_custom_transform_source(cli, datafiles):
# Set the project_dir alias in project.conf to the path to the tested project
project_config_path = os.path.join(project, "project.conf")
project_config = _yaml.load(project_config_path)
- aliases = _yaml.node_get(project_config, dict, "aliases")
+ aliases = project_config.get_mapping("aliases")
_yaml.node_set(aliases, "project_dir", "file://{}".format(project))
_yaml.dump(project_config, project_config_path)
diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
index bec4f9913..cd0f621d5 100644
--- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py
+++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
@@ -32,7 +32,7 @@ class FooTransformSource(Source):
def configure(self, node):
self.node_validate(node, ['ref', *Source.COMMON_CONFIG_KEYS])
- self.ref = self.node_get_member(node, str, 'ref', None)
+ self.ref = node.get_str('ref', None)
def preflight(self):
pass
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 2e1f72138..6ea6cc3e8 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -28,5 +28,4 @@ from .element_generators import create_element_size, update_element_size
from .junction import generate_junction
from .runner_integration import wait_for_cache_granularity
from .python_repo import setup_pypi_repo
-from .yaml import yaml_file_get_provenance
from .platform import override_platform_uname
diff --git a/tests/testutils/yaml.py b/tests/testutils/yaml.py
deleted file mode 100644
index a1dcb8646..000000000
--- a/tests/testutils/yaml.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Copyright (C) 2019 Bloomberg Finance LP
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-# Authors:
-# Angelos Evripiotis <jevripiotis@bloomberg.net>
-
-from buildstream import _yaml
-
-
-# yaml_file_get_provenance()
-#
-# Load a yaml file and return its _yaml.ProvenanceInformation object.
-#
-# This is useful for checking the provenance in BuildStream output is as
-# expected.
-#
-# Args:
-# path (str): The path to the file to be loaded
-# shortname (str): How the path should appear in the error
-# key (str): Optional key to look up in the loaded file
-# indices (list of indexes): Optional index path, in the case of list values
-#
-# Returns:
-# The ProvenanceInformation of the dict, member or list element
-#
-def yaml_file_get_provenance(path, shortname, key=None, indices=None):
- file_node = _yaml.load(path, shortname)
- if key:
- required_node = _yaml.node_get(file_node, dict, key, indices=indices)
- else:
- required_node = file_node
- provenance = _yaml.node_get_provenance(required_node)
- assert provenance is not None
- return provenance