Continue lazy provenance infotest_pyyaml

10 files changed, 100 insertions, 45 deletions

diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 48ab278c4..97e4a1fef 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -195,10 +195,13 @@ class ArtifactCache():
 
         artifacts = config_node.get('artifacts', [])
         if isinstance(artifacts, Mapping):
-            cache_specs.append(ArtifactCacheSpec._new_from_config_node(artifacts, basedir))
+            cache_specs.append(ArtifactCacheSpec._new_from_config_node(
+                _yaml.BstNode(config_node.bst_filename, artifacts, config_node.path + ('artifacts',)), basedir)
+            )
         elif isinstance(artifacts, list):
             for spec_node in artifacts:
-                cache_specs.append(ArtifactCacheSpec._new_from_config_node(spec_node, basedir))
+                cache_specs.append(ArtifactCacheSpec._new_from_config_node(
+                    _yaml.BstNode(config_node.bst_filename, spec_node, config_node.path + ('artifacts',)), basedir))
         else:
             provenance = _yaml.node_get_provenance(config_node, key='artifacts')
             raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
diff --git a/buildstream/_includes.py b/buildstream/_includes.py
index df14c9f2d..17e5e4b7a 100644
--- a/buildstream/_includes.py
+++ b/buildstream/_includes.py
@@ -120,7 +120,7 @@ class Includes:
                        included=set(),
                        current_loader=None,
                        only_local=False):
-        if isinstance(value, Mapping):
+        if isinstance(value, _yaml.BstNode):
             self.process(value,
                          included=included,
                          current_loader=current_loader,
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 11a3776fd..980392a67 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -184,6 +184,6 @@ def _extract_depends_from_node(node, *, key=None):
 
     # Now delete the field, we dont want it anymore
     if key in node:
-        del node[key]
+        del node.yaml_node[key]
 
     return output_deps
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 13761fb31..17909724c 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -453,12 +453,12 @@ class Loader():
         for i in range(len(sources)):
             source = _yaml.node_get(node, Mapping, Symbol.SOURCES, indices=[i])
             kind = _yaml.node_get(source, str, Symbol.KIND)
-            del source[Symbol.KIND]
+            del source.yaml_node[Symbol.KIND]
 
             # Directory is optional
             directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None)
             if directory:
-                del source[Symbol.DIRECTORY]
+                del source.yaml_node[Symbol.DIRECTORY]
 
             index = sources.index(source)
             meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py
index 3132af564..45da7fc28 100644
--- a/buildstream/_options/optionpool.py
+++ b/buildstream/_options/optionpool.py
@@ -187,7 +187,7 @@ class OptionPool():
         # and process any indirectly nested conditionals.
         #
         for _, value in _yaml.node_items(node):
-            if isinstance(value, Mapping):
+            if isinstance(value, _yaml.BstNode):
                 self.process_node(value)
             elif isinstance(value, list):
                 self._process_list(value)
@@ -238,7 +238,7 @@ class OptionPool():
     #
     def _process_list(self, values):
         for value in values:
-            if isinstance(value, Mapping):
+            if isinstance(value, _yaml.BstNode):
                 self.process_node(value)
             elif isinstance(value, list):
                 self._process_list(value)
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 3ec141d58..07db1b395 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -268,8 +268,10 @@ class Project():
     #
     def create_source(self, meta, *, first_pass=False):
         if first_pass:
+            print("FIRST PASS", meta.config)
             return self.first_pass_config.source_factory.create(self._context, self, meta)
         else:
+            print("SECOND", meta.config)
             return self.config.source_factory.create(self._context, self, meta)
 
     # get_alias_uri()
@@ -456,6 +458,8 @@ class Project():
                 raise
 
         pre_config_node = _yaml.node_copy(self._default_config_node)
+        print(pre_config_node.bst_filename)
+        print(self._project_conf.bst_filename)
         _yaml.composite(pre_config_node, self._project_conf)
 
         # Assert project's format version early, before validating toplevel keys
@@ -642,8 +646,8 @@ class Project():
         # assertion after.
         output.element_overrides = _yaml.node_get(config, Mapping, 'elements', default_value={})
         output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
-        config.pop('elements', None)
-        config.pop('sources', None)
+        config.yaml_node.pop('elements', None)
+        config.yaml_node.pop('sources', None)
         _yaml.node_final_assertions(config)
 
         self._load_plugin_factories(config, output)
diff --git a/buildstream/_variables.py b/buildstream/_variables.py
index 95b80cc08..c329475cf 100644
--- a/buildstream/_variables.py
+++ b/buildstream/_variables.py
@@ -43,7 +43,6 @@ _VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
 class Variables():
 
     def __init__(self, node):
-
         self.original = node
         self.variables = self._resolve(node)
 
@@ -147,8 +146,8 @@ class Variables():
                 unmatched += item_unmatched
 
             # Carry over provenance
-            resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
-            return (resolved, unmatched)
+            # resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
+            return (_yaml.BstNode(variables.bst_filename, resolved, variables.path), unmatched)
 
         # Resolve it until it's resolved or broken
         #
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index 966886790..4212265ce 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -178,6 +178,30 @@ class CompositeTypeError(CompositeError):
         self.actual_type = actual_type
 
 
+class BstNode(collections.abc.Mapping):
+    __slots__ = ('bst_filename', 'yaml_node', 'path')
+
+    def __init__(self, bst_filename, yaml_node, path):
+        self.yaml_node = yaml_node
+        self.bst_filename = bst_filename
+        self.path = path
+
+    def __getitem__(self, key):
+        return self.yaml_node.__getitem__(key)
+
+    def __setitem__(self, k, v):
+        self.yaml_node.__setitem__(k,v)
+
+    def __iter__(self):
+        return self.yaml_node.__iter__()
+
+    def __len__(self):
+        return self.yaml_node.__len__()
+
+    def __str__(self):
+        return str((self.yaml_node, self.bst_filename, self.path))
+
+
 # Loads a dictionary from some YAML
 #
 # Args:
@@ -240,7 +264,8 @@ def load_data(data, file=None, copy_tree=False):
                             .format(type(contents).__name__, file.name))
 
     # return node_decorated_copy(file, contents, copy_tree=copy_tree)
-    return contents
+    # return contents
+    return BstNode(file.name, contents, tuple())
 
 
 # Dumps a previously loaded YAML node to a file
@@ -329,14 +354,15 @@ def node_decorate_list(filename, target, source, toplevel):
 #
 def node_get_provenance(node, key=None, indices=None):
 
-    provenance = node.get(PROVENANCE_KEY)
-    if provenance and key:
-        provenance = provenance.members.get(key)
-        if provenance and indices is not None:
-            for index in indices:
-                provenance = provenance.elements[index]
+    # provenance = node.get(PROVENANCE_KEY)
+    # print(node, key, indices)
+    # if provenance and key:
+    #     provenance = provenance.members.get(key)
+    #     if provenance and indices is not None:
+    #         for index in indices:
+    #             provenance = provenance.elements[index]
 
-    return provenance
+    return node.bst_filename, node.path
 
 
 # A sentinel to be used as a default argument for functions that need
@@ -367,17 +393,18 @@ _sentinel = object()
 # Note:
 #    Returned strings are stripped of leading and trailing whitespace
 #
-def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
+def node_get(bst_node, expected_type, key, indices=None, *, default_value=_sentinel, allow_none=False):
+    node = bst_node.yaml_node
     value = node.get(key, default_value)
-    provenance = node_get_provenance(node)
     if value is _sentinel:
+        provenance = node_get_provenance(bst_node)
         raise LoadError(LoadErrorReason.INVALID_DATA,
                         "{}: Dictionary did not contain expected key '{}'".format(provenance, key))
 
     path = key
     if indices is not None:
         # Implied type check of the element itself
-        value = node_get(node, list, key)
+        value = node_get(bst_node, list, key)
         for index in indices:
             value = value[index]
             path += '[{:d}]'.format(index)
@@ -406,7 +433,7 @@ def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel,
             else:
                 raise ValueError()
         except (ValueError, TypeError):
-            provenance = node_get_provenance(node, key=key, indices=indices)
+            provenance = node_get_provenance(bst_node, key=key, indices=indices)
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}: Value of '{}' is not of the expected type '{}'"
                             .format(provenance, path, expected_type.__name__))
@@ -415,6 +442,8 @@ def node_get(node, expected_type, key, indices=None, *, default_value=_sentinel,
     if isinstance(value, str):
         value = value.strip()
 
+    if expected_type == collections.abc.Mapping:
+        return BstNode(bst_node.bst_filename, value, bst_node.path + (key,))
     return value
 
 
@@ -530,7 +559,10 @@ def node_items(node):
     for key, value in node.items():
         if key == PROVENANCE_KEY:
             continue
-        yield (key, value)
+        if isinstance(value, collections.abc.Mapping):
+            yield (key, BstNode(node.bst_filename, value, node.path + (key,)))
+        else:
+            yield (key, value)
 
 
 # Gives a node a dummy provenance, in case of compositing dictionaries
@@ -732,21 +764,19 @@ def composite_list(target_node, source_node, key):
     target_value = target_node.get(key)
     source_value = source_node[key]
 
-    target_key_provenance = node_get_provenance(target_node, key)
-    source_key_provenance = node_get_provenance(source_node, key)
-
     # Whenever a literal list is encountered in the source, it
     # overwrites the target values and provenance completely.
     #
     if isinstance(source_value, list):
 
-        source_provenance = node_get_provenance(source_node)
-        target_provenance = node_get_provenance(target_node)
+
 
         # Assert target type
         if not (target_value is None or
                 isinstance(target_value, list) or
                 is_composite_list(target_value)):
+            target_key_provenance = node_get_provenance(target_node, key)
+            source_key_provenance = node_get_provenance(source_node, key)
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}: List cannot overwrite value at: {}"
                             .format(source_key_provenance, target_key_provenance))
@@ -837,6 +867,9 @@ def composite_dict(target, source, path=None):
     # target_provenance = ensure_provenance(target)
     # source_provenance = ensure_provenance(source)
 
+    #source = bstnode_source.yaml_node
+    #target = bstnode_target.yaml_node
+
     for key, source_value in node_items(source):
 
         # Track the full path of keys, only for raising CompositeError
@@ -859,9 +892,9 @@ def composite_dict(target, source, path=None):
                 target[key] = target_value
 
                 # Give the new dict provenance
-                value_provenance = source_value.get(PROVENANCE_KEY)
-                if value_provenance:
-                    target_value[PROVENANCE_KEY] = value_provenance.clone()
+                # value_provenance = source_value.get(PROVENANCE_KEY)
+                # if value_provenance:
+                #    target_value[PROVENANCE_KEY] = value_provenance.clone()
 
                 # Add a new provenance member element to the containing dict
                 # target_provenance.members[key] = source_provenance.members[key]
@@ -890,13 +923,17 @@ def composite_dict(target, source, path=None):
 # Like composite_dict(), but raises an all purpose LoadError for convenience
 #
 def composite(target, source):
+    print(f"Merging\n\t{source.bst_filename} into\n\t{target.bst_filename}\n")
     assert hasattr(source, 'get')
 
-    source_provenance = node_get_provenance(source)
     try:
         composite_dict(target, source)
+        src = source.bst_filename if isinstance(source.bst_filename, tuple) else (source.bst_filename,)
+        trg = target.bst_filename if isinstance(target.bst_filename, tuple) else (target.bst_filename,)
+        target.bst_filename = trg + src
     except CompositeTypeError as e:
         error_prefix = ""
+        source_provenance = node_get_provenance(source)
         if source_provenance:
             error_prefix = "{}: ".format(source_provenance)
         raise LoadError(LoadErrorReason.ILLEGAL_COMPOSITE,
@@ -926,13 +963,15 @@ RoundTripRepresenter.add_representer(SanitizedDict,
 # Only dicts are ordered, list elements are left in order.
 #
 def node_sanitize(node):
+    if isinstance(node, BstNode):
+        node = node.yaml_node
 
     if isinstance(node, collections.abc.Mapping):
 
         result = SanitizedDict()
 
-        key_list = [key for key, _ in node_items(node)]
-        for key in sorted(key_list):
+        # key_list = [key for key, _ in node_items(node)]
+        for key in sorted(node.keys()):
             result[key] = node_sanitize(node[key])
 
         return result
@@ -957,11 +996,12 @@ def node_sanitize(node):
 #    LoadError: In the case that the specified node contained
 #               one or more invalid keys
 #
-def node_validate(node, valid_keys):
+def node_validate(bst_node, valid_keys):
+    node = bst_node.yaml_node
 
     # Probably the fastest way to do this: https://stackoverflow.com/a/23062482
     valid_keys = set(valid_keys)
-    valid_keys.add(PROVENANCE_KEY)
+    # valid_keys.add(PROVENANCE_KEY)
     invalid = next((key for key in node if key not in valid_keys), None)
 
     if invalid:
@@ -1069,6 +1109,9 @@ class ChainMap(collections.ChainMap):
 
 
 def node_chain_copy(source):
+    return BstNode(
+        source.bst_filename, deepcopy(source.yaml_node), source.path
+    )
     copy = ChainMap({}, source)
     for key, value in source.items():
         if isinstance(value, collections.abc.Mapping):
@@ -1096,7 +1139,10 @@ def list_chain_copy(source):
     return copy
 
 
-def node_copy(source):
+def node_copy(bst_node):
+    return BstNode(
+        bst_node.bst_filename, deepcopy(bst_node.yaml_node), bst_node.path
+    )
     copy = {}
     for key, value in source.items():
         if isinstance(value, collections.abc.Mapping):
diff --git a/buildstream/element.py b/buildstream/element.py
index a243826ed..7a7eb2b8b 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -2303,7 +2303,7 @@ class Element(Plugin):
         if not self.__defaults_set:
 
             # Load the plugin's accompanying .yaml file if one was provided
-            defaults = {}
+            defaults = _yaml.BstNode('', {}, tuple())
             try:
                 defaults = _yaml.load(plugin_conf, os.path.basename(plugin_conf))
             except LoadError as e:
@@ -2389,8 +2389,9 @@ class Element(Plugin):
         _yaml.node_final_assertions(variables)
 
         for var in ('project-name', 'element-name', 'max-jobs'):
-            provenance = _yaml.node_get_provenance(variables, var)
-            if provenance and provenance.filename != '':
+            #if provenance and provenance.filename != '':
+            if var in meta.variables.yaml_node:
+                provenance = _yaml.node_get_provenance(variables, var)
                 raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED,
                                 "{}: invalid redefinition of protected variable '{}'"
                                 .format(provenance, var))
diff --git a/buildstream/source.py b/buildstream/source.py
index 9e9bad71c..eaecec136 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -254,7 +254,7 @@ class Source(Plugin):
     All Sources derive from this class, this interface defines how
     the core will be interacting with Sources.
     """
-    __defaults = {}          # The defaults from the project
+    __defaults = _yaml.BstNode('', {}, tuple())          # The defaults from the project
     __defaults_set = False   # Flag, in case there are not defaults at all
 
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
@@ -1117,7 +1117,7 @@ class Source(Plugin):
                 sources = project.first_pass_config.source_overrides
             else:
                 sources = project.source_overrides
-            type(self).__defaults = sources.get(self.get_kind(), {})
+            type(self).__defaults = sources.get(self.get_kind(), _yaml.BstNode('', {}, tuple()))
             type(self).__defaults_set = True
 
     # This will resolve the final configuration to be handed
@@ -1127,6 +1127,8 @@ class Source(Plugin):
         config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={})
         config = _yaml.node_chain_copy(config)
 
+        print("copied config:", config)
+        print("meta config:", meta.config)
         _yaml.composite(config, meta.config)
         _yaml.node_final_assertions(config)