From 8f6e36acea77b35f454a53383a94b8fb838957fd Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Wed, 5 Jun 2019 21:16:50 +0100 Subject: _loader/types: cimport yaml functions for better speed - _yaml: export node_validate function as Cython, as it was not done before. This requires rewriting the function to remove a closure. - Optimize node check by not calling is_node(). --- setup.py | 2 +- src/buildstream/_loader/types.pyx | 16 ++++++++-------- src/buildstream/_yaml.pxd | 1 + src/buildstream/_yaml.pyx | 13 +++++++------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index e393096b8..ab3c6f30d 100755 --- a/setup.py +++ b/setup.py @@ -399,7 +399,7 @@ def register_cython_module(module_name, dependencies=None): BUILD_EXTENSIONS = [] register_cython_module("buildstream._loader._loader") -register_cython_module("buildstream._loader.types") +register_cython_module("buildstream._loader.types", dependencies=["buildstream._yaml"]) register_cython_module("buildstream._yaml") register_cython_module("buildstream._variables", dependencies=["buildstream._yaml"]) diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx index a9aa3ab32..5e4e12666 100644 --- a/src/buildstream/_loader/types.pyx +++ b/src/buildstream/_loader/types.pyx @@ -18,7 +18,7 @@ # Tristan Van Berkom from .._exceptions import LoadError, LoadErrorReason -from .. import _yaml +from .. cimport _yaml # Symbol(): @@ -51,14 +51,14 @@ class Symbol(): # A simple object describing a dependency # # Args: -# name (str): The element name +# name (str or Node): The element name # dep_type (str): The type of dependency, can be # Symbol.ALL, Symbol.BUILD, or Symbol.RUNTIME # junction (str): The element name of the junction, or None -# provenance (Provenance): The YAML node provenance of where this -# dependency was declared +# provenance (ProvenanceInformation): The YAML node provenance of where this +# dependency was declared # -class Dependency(): +class Dependency: def __init__(self, dep, provenance, default_dep_type=None): self.provenance = provenance @@ -67,7 +67,7 @@ class Dependency(): self.dep_type = default_dep_type self.junction = None - elif _yaml.is_node(dep): + elif type(dep) is _yaml.Node and type(dep.value) is dict: if default_dep_type: _yaml.node_validate(dep, ['filename', 'junction']) dep_type = default_dep_type @@ -75,7 +75,7 @@ class Dependency(): _yaml.node_validate(dep, ['filename', 'type', 'junction']) # Make type optional, for this we set it to None - dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None) + dep_type = _yaml.node_get(dep, str, Symbol.TYPE, None, None) if dep_type is None or dep_type == Symbol.ALL: dep_type = None elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]: @@ -86,7 +86,7 @@ class Dependency(): self.name = _yaml.node_get(dep, str, Symbol.FILENAME) self.dep_type = dep_type - self.junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None) + self.junction = _yaml.node_get(dep, str, Symbol.JUNCTION, None, None) else: raise LoadError(LoadErrorReason.INVALID_DATA, diff --git a/src/buildstream/_yaml.pxd b/src/buildstream/_yaml.pxd index 27a1a888e..6a12fa7b3 100644 --- a/src/buildstream/_yaml.pxd +++ b/src/buildstream/_yaml.pxd @@ -39,6 +39,7 @@ cdef class ProvenanceInformation: cpdef object node_get(Node node, object expected_type, str key, list indices=*, object default_value=*, bint allow_none=*) +cpdef void node_validate(Node node, list valid_keys) except * cpdef void node_set(Node node, object key, object value, list indices=*) except * cpdef list node_keys(object node) cpdef ProvenanceInformation node_get_provenance(Node node, str key=*, list indices=*) diff --git a/src/buildstream/_yaml.pyx b/src/buildstream/_yaml.pyx index 4505e2f95..f14b55e51 100644 --- a/src/buildstream/_yaml.pyx +++ b/src/buildstream/_yaml.pyx @@ -1119,16 +1119,17 @@ cpdef object node_sanitize(object node, object dict_type=OrderedDict): # LoadError: In the case that the specified node contained # one or more invalid keys # -def node_validate(Node node, list valid_keys): +cpdef void node_validate(Node node, list valid_keys) except *: # Probably the fastest way to do this: https://stackoverflow.com/a/23062482 cdef set valid_keys_set = set(valid_keys) - invalid = next((key for key in node.value if key not in valid_keys_set), None) + cdef str key - if invalid: - provenance = node_get_provenance(node, key=invalid) - raise LoadError(LoadErrorReason.INVALID_DATA, - "{}: Unexpected key: {}".format(provenance, invalid)) + for key in node.value: + if key not in valid_keys_set: + provenance = node_get_provenance(node, key=key) + raise LoadError(LoadErrorReason.INVALID_DATA, + "{}: Unexpected key: {}".format(provenance, key)) # Node copying -- cgit v1.2.1