_yaml.py: Add fast_load switch to enable loading with pyyamljennis/YAML_optimisations

* This also requires skipping ensure_provenance calls when necessary
* pyyaml has been added to the relevant requirement files

4 files changed, 52 insertions, 31 deletions

diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 677335404..5239ce2fc 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -176,6 +176,7 @@ def _extract_depends_from_node(node, *, key=None):
         output_deps.append(dependency)
 
     # Now delete the field, we dont want it anymore
-    del node[key]
+    if key in node:
+        del node[key]
 
     return output_deps
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 230094c22..80864f705 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -24,7 +24,8 @@ from copy import deepcopy
 from contextlib import ExitStack
 from pathlib import Path
 
-from ruamel import yaml
+import yaml  # this is pyyaml
+from ruamel import yaml as ruamelyaml
 from ruamel.yaml.representer import SafeRepresenter, RoundTripRepresenter
 from ruamel.yaml.constructor import RoundTripConstructor
 from ._exceptions import LoadError, LoadErrorReason
@@ -184,12 +185,13 @@ class CompositeTypeError(CompositeError):
 #    copy_tree (bool): Whether to make a copy, preserving the original toplevels
 #                      for later serialization
 #    yaml_cache (YamlCache): A yaml cache to consult rather than parsing
+#    fast_load (bool): whether to load the YAML with pyyaml (instead of ruamel.yaml)
 #
 # Returns (dict): A loaded copy of the YAML file with provenance information
 #
 # Raises: LoadError
 #
-def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=None):
+def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=None, fast_load=False):
     if not shortname:
         shortname = filename
 
@@ -203,7 +205,7 @@ def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=
             data, key = yaml_cache.get(project, filename, contents, copy_tree)
 
         if not data:
-            data = load_data(contents, file, copy_tree=copy_tree)
+            data = load_data(contents, file, copy_tree=copy_tree, fast_load=fast_load)
             if yaml_cache:
                 yaml_cache.put_from_key(project, filename, key, data)
 
@@ -219,13 +221,22 @@ def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=
 
 # Like load(), but doesnt require the data to be in a file
 #
-def load_data(data, file=None, copy_tree=False):
+def load_data(data, file=None, copy_tree=False, fast_load=True):
 
-    try:
-        contents = yaml.load(data, yaml.loader.RoundTripLoader, preserve_quotes=True)
-    except (yaml.scanner.ScannerError, yaml.composer.ComposerError, yaml.parser.ParserError) as e:
-        raise LoadError(LoadErrorReason.INVALID_YAML,
-                        "Malformed YAML:\n\n{}\n\n{}\n".format(e.problem, e.problem_mark)) from e
+    if fast_load:
+        try:
+            contents = yaml.load(data, Loader=yaml.BaseLoader)  # BaseLoader returns strings... There is also a CBaseLoader for speed
+        except yaml.YAMLError as e:
+            mark = e.problem_mark
+            raise LoadError(LoadErrorReason.INVALID_YAML,
+                            "{}:\nMalformed YAML: Line {}, Column: {}".format(file.name, mark.line+1, mark.column+1))
+
+    else:
+        try:
+            contents = ruamelyaml.load(data, ruamelyaml.loader.RoundTripLoader, preserve_quotes=True)
+        except (ruamelyaml.scanner.ScannerError, ruamelyaml.composer.ComposerError, ruamelyaml.parser.ParserError) as e:
+            raise LoadError(LoadErrorReason.INVALID_YAML,
+                            "Malformed YAML:\n\n{}\n\n{}\n".format(e.problem, e.problem_mark)) from e
 
     if not isinstance(contents, dict):
         # Special case allowance for None, when the loaded file has only comments in it.
@@ -236,7 +247,10 @@ def load_data(data, file=None, copy_tree=False):
                             "YAML file has content of type '{}' instead of expected type 'dict': {}"
                             .format(type(contents).__name__, file.name))
 
-    return node_decorated_copy(file, contents, copy_tree=copy_tree)
+    if fast_load:
+        return contents
+    else:
+        return node_decorated_copy(file, contents, copy_tree=copy_tree)
 
 
 # Dumps a previously loaded YAML node to a file
@@ -252,7 +266,7 @@ def dump(node, filename=None):
             f = stack.enter_context(utils.save_file_atomic(filename, 'w'))
         else:
             f = sys.stdout
-        yaml.round_trip_dump(node, f)
+        ruamelyaml.round_trip_dump(node, f)
 
 
 # node_decorated_copy()
@@ -554,7 +568,7 @@ def is_ruamel_str(value):
 
     if isinstance(value, str):
         return True
-    elif isinstance(value, yaml.scalarstring.ScalarString):
+    elif isinstance(value, ruamelyaml.scalarstring.ScalarString):
         return True
 
     return False
@@ -829,9 +843,10 @@ def composite_list(target_node, source_node, key):
 # This is useful for overriding configuration files and element
 # configurations.
 #
-def composite_dict(target, source, path=None):
-    target_provenance = ensure_provenance(target)
-    source_provenance = ensure_provenance(source)
+def composite_dict(target, source, path=None, skip_provenance=False):
+    if not skip_provenance:
+        target_provenance = ensure_provenance(target)
+        source_provenance = ensure_provenance(source)
 
     for key, source_value in node_items(source):
 
@@ -854,19 +869,20 @@ def composite_dict(target, source, path=None):
                 target_value = {}
                 target[key] = target_value
 
-                # Give the new dict provenance
-                value_provenance = source_value.get(PROVENANCE_KEY)
-                if value_provenance:
-                    target_value[PROVENANCE_KEY] = value_provenance.clone()
+                if not skip_provenance:
+                    # Give the new dict provenance
+                    value_provenance = source_value.get(PROVENANCE_KEY)
+                    if value_provenance:
+                        target_value[PROVENANCE_KEY] = value_provenance.clone()
 
-                # Add a new provenance member element to the containing dict
-                target_provenance.members[key] = source_provenance.members[key]
+                        # Add a new provenance member element to the containing dict
+                        target_provenance.members[key] = source_provenance.members[key]
 
             if not isinstance(target_value, collections.abc.Mapping):
                 raise CompositeTypeError(thispath, type(target_value), type(source_value))
 
             # Recurse into matching dictionary
-            composite_dict(target_value, source_value, path=thispath)
+            composite_dict(target_value, source_value, path=thispath, skip_provenance=skip_provenance)
 
         else:
 
@@ -879,7 +895,8 @@ def composite_dict(target, source, path=None):
                     raise CompositeTypeError(thispath, type(target_value), type(source_value))
 
             # Overwrite simple values, lists and mappings have already been handled
-            target_provenance.members[key] = source_provenance.members[key].clone()
+            if not skip_provenance:
+                target_provenance.members[key] = source_provenance.members[key].clone()
             target[key] = source_value
 
 
@@ -890,7 +907,7 @@ def composite(target, source):
 
     source_provenance = node_get_provenance(source)
     try:
-        composite_dict(target, source)
+        composite_dict(target, source, skip_provenance=source_provenance is None)
     except CompositeTypeError as e:
         error_prefix = ""
         if source_provenance:
@@ -1095,15 +1112,15 @@ class ChainMap(collections.ChainMap):
 # retrieve them from the source node when needed.  Other copiers might copy
 # them, so we call them __QUICK_TYPES.
 __QUICK_TYPES = (str, bool,
-                 yaml.scalarstring.PreservedScalarString,
-                 yaml.scalarstring.SingleQuotedScalarString,
-                 yaml.scalarstring.DoubleQuotedScalarString)
+                 ruamelyaml.scalarstring.PreservedScalarString,
+                 ruamelyaml.scalarstring.SingleQuotedScalarString,
+                 ruamelyaml.scalarstring.DoubleQuotedScalarString)
 
 # These types have to be iterated like a dictionary
-__DICT_TYPES = (dict, ChainMap, yaml.comments.CommentedMap)
+__DICT_TYPES = (dict, ChainMap, ruamelyaml.comments.CommentedMap)
 
 # These types have to be iterated like a list
-__LIST_TYPES = (list, yaml.comments.CommentedSeq)
+__LIST_TYPES = (list, ruamelyaml.comments.CommentedSeq)
 
 # These are the provenance types, which have to be cloned rather than any other
 # copying tactic.
@@ -1165,7 +1182,8 @@ def node_copy(source):
         else:
             raise ValueError("Unable to be quick about node_copy of {}".format(value_type))
 
-    ensure_provenance(copy)
+    #ensure_provenance(source)
+    # Let's not ensure that...
 
     return copy
 
diff --git a/requirements/requirements.in b/requirements/requirements.in
index 4f08969bb..52df032c8 100644
--- a/requirements/requirements.in
+++ b/requirements/requirements.in
@@ -12,6 +12,7 @@ psutil
 #
 # See issues #571 and #790.
 ruamel.yaml >= 0.15.41, < 0.15.52
+pyyaml
 setuptools
 pyroaring
 ujson
diff --git a/requirements/requirements.txt b/requirements/requirements.txt
index d46f92553..aaf787d85 100644
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
@@ -14,6 +14,7 @@ psutil==5.4.8
 ruamel.yaml==0.15.51
 setuptools==39.0.1
 pyroaring==0.2.6
+pyyaml==3.13
 ujson==1.35
 ## The following requirements were added by pip freeze:
 MarkupSafe==1.1.0