summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/buildstream/_basecache.py2
-rw-r--r--src/buildstream/_gitsourcebase.py4
-rw-r--r--src/buildstream/_includes.py2
-rw-r--r--src/buildstream/_loader/loader.py8
-rw-r--r--src/buildstream/_loader/types.pyx14
-rw-r--r--src/buildstream/_options/optionenum.py2
-rw-r--r--src/buildstream/_options/optionflags.py6
-rw-r--r--src/buildstream/_options/optionpool.py2
-rw-r--r--src/buildstream/_plugincontext.py3
-rw-r--r--src/buildstream/_project.py26
-rw-r--r--src/buildstream/_projectrefs.py6
-rw-r--r--src/buildstream/_yaml.pxd7
-rw-r--r--src/buildstream/_yaml.pyx53
-rw-r--r--src/buildstream/buildelement.py13
-rw-r--r--src/buildstream/element.py11
15 files changed, 106 insertions, 53 deletions
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index faf4cc878..64ee8fe57 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -77,7 +77,7 @@ class BaseCache():
artifacts = [config_node.get_mapping(cls.config_node_name)]
except LoadError:
try:
- artifacts = _yaml.node_get(config_node, list, cls.config_node_name, default_value=[])
+ artifacts = config_node.get_sequence(cls.config_node_name, default=[])
except LoadError:
provenance = _yaml.node_get_provenance(config_node, key=cls.config_node_name)
raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 61ecd36eb..4837877f4 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -383,7 +383,7 @@ class _GitSourceBase(Source):
'track-tags', 'tags']
self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
- tags_node = self.node_get_member(node, list, 'tags', [])
+ tags_node = node.get_sequence('tags', [])
for tag_node in tags_node:
self.node_validate(tag_node, ['tag', 'commit', 'annotated'])
@@ -663,7 +663,7 @@ class _GitSourceBase(Source):
def _load_tags(self, node):
tags = []
- tags_node = self.node_get_member(node, list, 'tags', [])
+ tags_node = node.get_sequence('tags', [])
for tag_node in tags_node:
tag = tag_node.get_str('tag')
commit_ref = tag_node.get_str('commit')
diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py
index 133cf50d0..9bed8189c 100644
--- a/src/buildstream/_includes.py
+++ b/src/buildstream/_includes.py
@@ -41,7 +41,7 @@ class Includes:
includes = [includes]
except LoadError:
try:
- includes = _yaml.node_get(node, list, '(@)')
+ includes = node.get_sequence('(@)').as_str_list()
except LoadError:
provenance = _yaml.node_get_provenance(node, key='(@)')
raise LoadError(LoadErrorReason.INVALID_DATA,
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 5f533731b..64e7fafd0 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -466,13 +466,12 @@ class Loader():
elt_provenance = _yaml.node_get_provenance(node)
meta_sources = []
- sources = _yaml.node_get(node, list, Symbol.SOURCES, default_value=[])
+ sources = node.get_sequence(Symbol.SOURCES, default=[])
element_kind = node.get_str(Symbol.KIND)
# Safe loop calling into _yaml.node_get() for each element ensures
# we have good error reporting
- for i in range(len(sources)):
- source = _yaml.node_get(node, dict, Symbol.SOURCES, indices=[i])
+ for index, source in enumerate(sources):
kind = source.get_str(Symbol.KIND)
_yaml.node_del(source, Symbol.KIND)
@@ -481,7 +480,6 @@ class Loader():
if directory:
_yaml.node_del(source, Symbol.DIRECTORY)
- index = sources.index(source)
meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
meta_sources.append(meta_source)
@@ -490,7 +488,7 @@ class Loader():
node.get_mapping(Symbol.CONFIG, default={}),
node.get_mapping(Symbol.VARIABLES, default={}),
node.get_mapping(Symbol.ENVIRONMENT, default={}),
- _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
+ node.get_sequence(Symbol.ENV_NOCACHE, default=[]).as_str_list(),
node.get_mapping(Symbol.PUBLIC, default={}),
node.get_mapping(Symbol.SANDBOX, default={}),
element_kind == 'junction')
diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx
index a3e49b040..dfd8f9046 100644
--- a/src/buildstream/_loader/types.pyx
+++ b/src/buildstream/_loader/types.pyx
@@ -72,8 +72,8 @@ cdef class Dependency:
self.provenance = provenance
- if type(dep) is str:
- self.name = <str> dep
+ if type(dep) is _yaml.ScalarNode:
+ self.name = dep.as_str()
self.dep_type = default_dep_type
self.junction = None
@@ -138,15 +138,15 @@ cdef class Dependency:
# acc (list): a list in which to add the loaded dependencies
#
cdef void _extract_depends_from_node(_yaml.Node node, str key, str default_dep_type, list acc) except *:
- cdef list depends = <list> _yaml.node_get(node, list, key, None, [])
- cdef int index
+ cdef _yaml.SequenceNode depends = node.get_sequence(key, [])
+ cdef _yaml.Node dep_node
cdef _yaml.ProvenanceInformation dep_provenance
- for index in range(len(depends)):
+ for dep_node in depends:
# FIXME: the provenance information would be obtainable from the Node directly if we stop
# stripping provenance and have proper nodes for str elements
- dep_provenance = <_yaml.ProvenanceInformation> _yaml.node_get_provenance(node, key=key, indices=[index])
- dependency = Dependency(depends[index], dep_provenance, default_dep_type=default_dep_type)
+ dep_provenance = <_yaml.ProvenanceInformation> _yaml.node_get_provenance(dep_node)
+ dependency = Dependency(dep_node, dep_provenance, default_dep_type=default_dep_type)
acc.append(dependency)
# Now delete the field, we dont want it anymore
diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py
index 04a19395b..f04cecd8b 100644
--- a/src/buildstream/_options/optionenum.py
+++ b/src/buildstream/_options/optionenum.py
@@ -46,7 +46,7 @@ class OptionEnum(Option):
_yaml.node_validate(node, valid_symbols)
- self.values = _yaml.node_get(node, list, 'values', default_value=[])
+ self.values = node.get_sequence('values', default=[]).as_str_list()
if not self.values:
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: No values specified for {} option '{}'"
diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py
index eba3a8dd5..c9758e403 100644
--- a/src/buildstream/_options/optionflags.py
+++ b/src/buildstream/_options/optionflags.py
@@ -53,11 +53,11 @@ class OptionFlags(Option):
"{}: No values specified for {} option '{}'"
.format(_yaml.node_get_provenance(node), self.OPTION_TYPE, self.name))
- self.value = _yaml.node_get(node, list, 'default', default_value=[])
+ self.value = node.get_sequence('default', default=[]).as_str_list()
self.validate(self.value, _yaml.node_get_provenance(node, 'default'))
def load_value(self, node, *, transform=None):
- self.value = _yaml.node_get(node, list, self.name)
+ self.value = node.get_sequence(self.name).as_str_list()
if transform:
self.value = [transform(x) for x in self.value]
self.value = sorted(self.value)
@@ -90,4 +90,4 @@ class OptionFlags(Option):
def load_valid_values(self, node):
# Allow the more descriptive error to raise when no values
# exist rather than bailing out here (by specifying default_value)
- return _yaml.node_get(node, list, 'values', default_value=[])
+ return node.get_sequence('values', default=[]).as_str_list()
diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py
index 25b96fa98..9a4a88c36 100644
--- a/src/buildstream/_options/optionpool.py
+++ b/src/buildstream/_options/optionpool.py
@@ -248,7 +248,7 @@ class OptionPool():
# Return true if a conditional was processed.
#
def _process_one_node(self, node):
- conditions = _yaml.node_get(node, list, '(?)', default_value=None)
+ conditions = node.get_sequence('(?)', default=None)
assertion = node.get_str('(!)', default=None)
# Process assersions first, we want to abort on the first encountered
diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py
index a6f5e08dc..2442e306f 100644
--- a/src/buildstream/_plugincontext.py
+++ b/src/buildstream/_plugincontext.py
@@ -22,7 +22,6 @@ import inspect
from ._exceptions import PluginError, LoadError, LoadErrorReason
from . import utils
-from . import _yaml
# A Context for loading plugin types
@@ -138,7 +137,7 @@ class PluginContext():
loaded_dependency = False
for origin in self._plugin_origins:
- if kind not in _yaml.node_get(origin, list, 'plugins'):
+ if kind not in origin.get_sequence('plugins').as_str_list():
continue
if origin.get_str('origin') == 'local':
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 7b498820b..ad87a138e 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -602,10 +602,10 @@ class Project():
defaults = pre_config_node.get_mapping('defaults')
_yaml.node_validate(defaults, ['targets'])
- self._default_targets = _yaml.node_get(defaults, list, "targets")
+ self._default_targets = defaults.get_sequence("targets").as_str_list()
# Fatal warnings
- self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[])
+ self._fatal_warnings = pre_config_node.get_sequence('fatal-warnings', default=[]).as_str_list()
self.loader = Loader(self._context, self,
parent=parent_loader, fetch_subprojects=fetch_subprojects)
@@ -678,7 +678,7 @@ class Project():
# Load sandbox environment variables
self.base_environment = config.get_mapping('environment')
- self.base_env_nocache = _yaml.node_get(config, list, 'environment-nocache')
+ self.base_env_nocache = config.get_sequence('environment-nocache').as_str_list()
# Load sandbox configuration
self._sandbox = config.get_mapping('sandbox')
@@ -710,7 +710,7 @@ class Project():
# Parse shell options
shell_options = config.get_mapping('shell')
_yaml.node_validate(shell_options, ['command', 'environment', 'host-files'])
- self._shell_command = _yaml.node_get(shell_options, list, 'command')
+ self._shell_command = shell_options.get_sequence('command').as_str_list()
# Perform environment expansion right away
shell_environment = shell_options.get_mapping('environment', default={})
@@ -719,20 +719,18 @@ class Project():
self._shell_environment[key] = os.path.expandvars(value)
# Host files is parsed as a list for convenience
- host_files = _yaml.node_get(shell_options, list, 'host-files', default_value=[])
+ host_files = shell_options.get_sequence('host-files', default=[])
for host_file in host_files:
- if isinstance(host_file, str):
+ if isinstance(host_file, _yaml.ScalarNode):
mount = HostMount(host_file)
else:
# Some validation
- index = host_files.index(host_file)
- host_file_desc = _yaml.node_get(shell_options, dict, 'host-files', indices=[index])
- _yaml.node_validate(host_file_desc, ['path', 'host_path', 'optional'])
+ _yaml.node_validate(host_file, ['path', 'host_path', 'optional'])
# Parse the host mount
- path = host_file_desc.get_str('path')
- host_path = host_file_desc.get_str('host_path', default=None)
- optional = _yaml.node_get(host_file_desc, bool, 'optional', default_value=False)
+ path = host_file.get_str('path')
+ host_path = host_file.get_str('host_path', default=None)
+ optional = host_file.get_bool('optional', default=False)
mount = HostMount(path, host_path, optional)
self._shell_host_files.append(mount)
@@ -809,7 +807,7 @@ class Project():
output.default_mirror = self._default_mirror or overrides.get_str(
'default-mirror', default=None)
- mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+ mirrors = config.get_sequence('mirrors', default=[])
for mirror in mirrors:
allowed_mirror_fields = [
'name', 'aliases'
@@ -869,7 +867,7 @@ class Project():
plugin_element_origins = [] # Origins of custom elements
# Plugin origins and versions
- origins = _yaml.node_get(config, list, 'plugins', default_value=[])
+ origins = config.get_sequence('plugins', default=[])
source_format_versions = {}
element_format_versions = {}
for origin in origins:
diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py
index 847935c0b..f296858cf 100644
--- a/src/buildstream/_projectrefs.py
+++ b/src/buildstream/_projectrefs.py
@@ -133,16 +133,16 @@ class ProjectRefs():
# Fetch the element
try:
- element_list = _yaml.node_get(project_node, list, element)
+ element_list = project_node.get_sequence(element)
except LoadError:
if not ensure:
return None
- element_list = []
+ element_list = _yaml.new_empty_list_node()
_yaml.node_set(project_node, element, element_list)
# Fetch the source index
try:
- node = element_list[source_index]
+ node = element_list.mapping_at(source_index)
except IndexError:
if not ensure:
return None
diff --git a/src/buildstream/_yaml.pxd b/src/buildstream/_yaml.pxd
index 146bec0fc..91fc1180d 100644
--- a/src/buildstream/_yaml.pxd
+++ b/src/buildstream/_yaml.pxd
@@ -32,6 +32,7 @@ cdef class MappingNode(Node):
cdef Node get(self, str key, default, default_constructor)
cpdef MappingNode get_mapping(self, str key, default=*)
cpdef ScalarNode get_scalar(self, str key, default=*)
+ cpdef SequenceNode get_sequence(self, str key, object default=*)
cpdef bint get_bool(self, str key, default=*) except *
cpdef int get_int(self, str key, default=*) except *
cpdef str get_str(self, str key, object default=*)
@@ -44,6 +45,12 @@ cdef class ScalarNode(Node):
cpdef bint is_none(self)
+cdef class SequenceNode(Node):
+ cpdef MappingNode mapping_at(self, int index)
+ cpdef SequenceNode sequence_at(self, int index)
+ cpdef list as_str_list(self)
+
+
cdef class ProvenanceInformation:
cdef public Node node
diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx
index 2c1714158..a2327dbc5 100644
--- a/src/buildstream/_yaml.pyx
+++ b/src/buildstream/_yaml.pyx
@@ -170,6 +170,17 @@ cdef class MappingNode(Node):
return value
+ cpdef SequenceNode get_sequence(self, str key, object default=_sentinel):
+ value = self.get(key, default, SequenceNode)
+
+ if type(value) is not SequenceNode and value is not None:
+ provenance = node_get_provenance(value)
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type 'Sequence'"
+ .format(provenance, key))
+
+ return value
+
cpdef bint get_bool(self, str key, object default=_sentinel) except *:
cdef ScalarNode scalar = self.get_scalar(key, default)
return scalar.as_bool()
@@ -183,13 +194,48 @@ cdef class MappingNode(Node):
return scalar.as_str()
-class SequenceNode(Node):
+cdef class SequenceNode(Node):
def __init__(self, list value, int file_index, int line, int column):
self.value = value
self.file_index = file_index
self.line = line
self.column = column
+ cpdef MappingNode mapping_at(self, int index):
+ value = self.value[index]
+
+ if type(value) is not MappingNode:
+ provenance = node_get_provenance(self)
+ path = ["[{}]".format(p) for p in node_find_target(provenance, self)] + ["[{}]".format(index)]
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type '{}'"
+ .format(provenance, path, MappingNode.__name__))
+ return value
+
+ cpdef SequenceNode sequence_at(self, int index):
+ value = self.value[index]
+
+ if type(value) is not SequenceNode:
+ provenance = node_get_provenance(self)
+ path = ["[{}]".format(p) for p in node_find_target(provenance, self)] + ["[{}]".format(index)]
+ raise LoadError(LoadErrorReason.INVALID_DATA,
+ "{}: Value of '{}' is not of the expected type '{}'"
+ .format(provenance, path, SequenceNode.__name__))
+
+ return value
+
+ cpdef list as_str_list(self):
+ return [node.as_str() for node in self.value]
+
+ def __iter__(self):
+ return iter(self.value)
+
+ def __len__(self):
+ return len(self.value)
+
+ def __reversed__(self):
+ return reversed(self.value)
+
# Metadata container for a yaml toplevel node.
#
@@ -935,6 +981,11 @@ def new_empty_node(Node ref_node=None):
return MappingNode({}, _SYNTHETIC_FILE_INDEX, 0, 0)
+# FIXME: we should never need that
+def new_empty_list_node():
+ return SequenceNode([], _SYNTHETIC_FILE_INDEX, 0, 0)
+
+
# new_node_from_dict()
#
# Args:
diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py
index 158f5fc11..48be1b83a 100644
--- a/src/buildstream/buildelement.py
+++ b/src/buildstream/buildelement.py
@@ -281,14 +281,11 @@ class BuildElement(Element):
# Private Local Methods #
#############################################################
def __get_commands(self, node, name):
- list_node = self.node_get_member(node, list, name, [])
- commands = []
-
- for i in range(len(list_node)):
- command = self.node_subst_list_element(node, name, [i])
- commands.append(command)
-
- return commands
+ raw_commands = node.get_sequence(name, []).as_str_list()
+ return [
+ self.substitute_variables(command)
+ for command in raw_commands
+ ]
def __run_command(self, sandbox, cmd):
# Note the -e switch to 'sh' means to exit with an error
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 606213e74..d2139b08b 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -484,6 +484,9 @@ class Element(Plugin):
return None
+ def substitute_variables(self, value):
+ return self.__variables.subst(value)
+
def node_subst_member(self, node, member_name, default=_yaml._sentinel):
"""Fetch the value of a string node member, substituting any variables
in the loaded value with the element contextual variables.
@@ -853,9 +856,9 @@ class Element(Plugin):
if bstdata is not None:
with sandbox.batch(SandboxFlags.NONE):
- commands = self.node_get_member(bstdata, list, 'integration-commands', [])
- for i in range(len(commands)):
- cmd = self.node_subst_list_element(bstdata, 'integration-commands', [i])
+ commands = bstdata.get_sequence('integration-commands', []).as_str_list()
+ for command in commands:
+ cmd = self.substitute_variables(command)
sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/',
label=cmd)
@@ -2619,7 +2622,7 @@ class Element(Plugin):
else:
project_nocache = project.base_env_nocache
- default_nocache = _yaml.node_get(cls.__defaults, list, 'environment-nocache', default_value=[])
+ default_nocache = cls.__defaults.get_sequence('environment-nocache', default=[]).as_str_list()
element_nocache = meta.env_nocache
# Accumulate values from the element default, the project and the element
View Lorry Controller Status