Untangling _yaml and utils modules.

These had a circular import, which is only supported > python 3.5
but undesirable anyway.

6 files changed, 76 insertions, 77 deletions

diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index fc86c76ba..e195d3df8 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -33,7 +33,7 @@ from ..exceptions import _BstError
 from .._message import MessageType, unconditional_messages
 from .._pipeline import Pipeline, PipelineError
 from .._scheduler import Scheduler
-from .. import utils
+from .. import utils, _yaml
 from .._profile import Topics, profile_start, profile_end
 
 # Import frontend assets
@@ -278,28 +278,28 @@ def show(app, target, arch, variant, deps, order, format):
 
         # Element configuration
         if "%{config" in format:
-            config = utils._node_sanitize(element._Element__config)
+            config = _yaml.node_sanitize(element._Element__config)
             line = p.fmt_subst(
                 line, 'config',
                 yaml.round_trip_dump(config, default_flow_style=False, allow_unicode=True))
 
         # Variables
         if "%{vars" in format:
-            variables = utils._node_sanitize(element._Element__variables.variables)
+            variables = _yaml.node_sanitize(element._Element__variables.variables)
             line = p.fmt_subst(
                 line, 'vars',
                 yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True))
 
         # Environment
         if "%{env" in format:
-            environment = utils._node_sanitize(element._Element__environment)
+            environment = _yaml.node_sanitize(element._Element__environment)
             line = p.fmt_subst(
                 line, 'env',
                 yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True))
 
         # Public
         if "%{public" in format:
-            environment = utils._node_sanitize(element._Element__public)
+            environment = _yaml.node_sanitize(element._Element__public)
             line = p.fmt_subst(
                 line, 'public',
                 yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True))
diff --git a/buildstream/_scheduler.py b/buildstream/_scheduler.py
index c2244d3a6..ca054f887 100644
--- a/buildstream/_scheduler.py
+++ b/buildstream/_scheduler.py
@@ -31,7 +31,7 @@ from ruamel import yaml
 from ._message import Message, MessageType, unconditional_messages
 from .exceptions import _BstError
 from .plugin import _plugin_lookup
-from . import utils
+from . import utils, _yaml
 
 
 # Process class that doesn't call waitpid on its own.
@@ -588,7 +588,7 @@ class Job():
             # Print the element's environment at the beginning of any element's log file.
             #
             # This should probably be omitted for non-build tasks but it's harmless here
-            elt_env = utils._node_sanitize(element._Element__environment)
+            elt_env = _yaml.node_sanitize(element._Element__environment)
             env_dump = yaml.round_trip_dump(elt_env, default_flow_style=False, allow_unicode=True)
             self.message(element, MessageType.LOG,
                          "Build environment for element {}".format(element.name),
diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py
index c74d401ff..70f782c12 100644
--- a/buildstream/_yaml.py
+++ b/buildstream/_yaml.py
@@ -24,7 +24,6 @@ from enum import Enum
 
 from ruamel import yaml
 from . import ImplError, LoadError, LoadErrorReason
-from . import utils
 
 
 # We store information in the loaded yaml on a DictProvenance
@@ -463,7 +462,7 @@ def composite_dict(target, source, policy=CompositePolicy.OVERWRITE, typesafe=Fa
                 # Ensure target has only copies of mutable source values
                 if (isinstance(target_value, list) and
                     isinstance(source_value, list)):
-                    target[key] = utils._list_chain_copy(source_value)
+                    target[key] = list_chain_copy(source_value)
                 else:
                     target[key] = source_value
 
@@ -473,7 +472,7 @@ def composite_dict(target, source, policy=CompositePolicy.OVERWRITE, typesafe=Fa
                     isinstance(source_value, list)):
 
                     # Ensure target has only copies of mutable source values
-                    target[key] += utils._list_chain_copy(source_value)
+                    target[key] += list_chain_copy(source_value)
 
                     # Append element provenances from source list to target
                     target_list_provenance = target_provenance.members[key]
@@ -514,3 +513,57 @@ def composite(target, source, policy=CompositePolicy.OVERWRITE, typesafe=False):
                          e.expected_type.__name__,
                          e.path,
                          e.actual_type.__name__)) from e
+
+
+# node_sanitize()
+#
+# Returnes an alphabetically ordered recursive copy
+# of the source node with internal provenance information stripped.
+#
+# Only dicts are ordered, list elements are left in order.
+#
+def node_sanitize(node):
+
+    if isinstance(node, collections.Mapping):
+
+        result = collections.OrderedDict()
+
+        for key in sorted(node):
+            if key == PROVENANCE_KEY:
+                continue
+            result[key] = node_sanitize(node[key])
+
+        return result
+
+    elif isinstance(node, list):
+        return [node_sanitize(elt) for elt in node]
+
+    return node
+
+
+def node_chain_copy(source):
+    copy = collections.ChainMap({}, source)
+    for key, value in source.items():
+        if isinstance(value, collections.Mapping):
+            copy[key] = node_chain_copy(value)
+        elif isinstance(value, list):
+            copy[key] = list_chain_copy(value)
+        elif isinstance(value, Provenance):
+            copy[key] = value.clone()
+
+    return copy
+
+
+def list_chain_copy(source):
+    copy = []
+    for item in source:
+        if isinstance(item, collections.Mapping):
+            copy.append(node_chain_copy(item))
+        elif isinstance(item, list):
+            copy.append(list_chain_copy(item))
+        elif isinstance(item, Provenance):
+            copy.append(item.clone())
+        else:
+            copy.append(item)
+
+    return copy
diff --git a/buildstream/element.py b/buildstream/element.py
index 772e5fc61..091ed3885 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -423,7 +423,7 @@ class Element(Plugin):
            (dict): A dictionary of string key/values suitable for passing
            to :func:`Sandbox.run() <buildstream.sandbox.Sandbox.run>`
         """
-        return utils._node_sanitize(self.__environment)
+        return _yaml.node_sanitize(self.__environment)
 
     #############################################################
     #                  Abstract Element Methods                 #
@@ -853,7 +853,7 @@ class Element(Plugin):
 
     def __compose_default_splits(self, defaults):
         project = self.get_project()
-        project_splits = utils._node_chain_copy(project._splits)
+        project_splits = _yaml.node_chain_copy(project._splits)
 
         element_public = _yaml.node_get(defaults, Mapping, 'public', default_value={})
         element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={})
@@ -910,7 +910,7 @@ class Element(Plugin):
         project = self.get_project()
         default_env = _yaml.node_get(self.__defaults, Mapping, 'environment', default_value={})
 
-        environment = utils._node_chain_copy(project._environment)
+        environment = _yaml.node_chain_copy(project._environment)
         _yaml.composite(environment, default_env, typesafe=True)
         _yaml.composite(environment, meta.environment, typesafe=True)
 
@@ -941,7 +941,7 @@ class Element(Plugin):
         project = self.get_project()
         default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
 
-        variables = utils._node_chain_copy(project._variables)
+        variables = _yaml.node_chain_copy(project._variables)
         _yaml.composite(variables, default_vars, typesafe=True)
         _yaml.composite(variables, meta.variables, typesafe=True)
 
@@ -954,7 +954,7 @@ class Element(Plugin):
 
         # The default config is already composited with the project overrides
         config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={})
-        config = utils._node_chain_copy(config)
+        config = _yaml.node_chain_copy(config)
 
         _yaml.composite(config, meta.config, typesafe=True)
 
@@ -965,12 +965,12 @@ class Element(Plugin):
     #
     def __extract_public(self, meta):
         base_public = _yaml.node_get(self.__defaults, Mapping, 'public', default_value={})
-        base_public = utils._node_chain_copy(base_public)
+        base_public = _yaml.node_chain_copy(base_public)
 
         base_bst = _yaml.node_get(base_public, Mapping, 'bst', default_value={})
         base_splits = _yaml.node_get(base_bst, Mapping, 'split-rules', default_value={})
 
-        element_public = utils._node_chain_copy(meta.public)
+        element_public = _yaml.node_chain_copy(meta.public)
         element_bst = _yaml.node_get(element_public, Mapping, 'bst', default_value={})
         element_splits = _yaml.node_get(element_bst, Mapping, 'split-rules', default_value={})
 
diff --git a/buildstream/utils.py b/buildstream/utils.py
index a4bd90be5..bb9e18444 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -29,9 +29,9 @@ import pickle
 import calendar
 import signal
 from contextlib import contextmanager
-from collections import OrderedDict, ChainMap, deque
-from . import _yaml
+from collections import deque
 from . import ProgramNotFoundError
+from . import _yaml
 
 
 def list_relative_paths(directory):
@@ -518,65 +518,11 @@ def _relative_symlink_target(root, symlink, target):
 #    (str): An sha256 hex digest of the given value
 #
 def _generate_key(value):
-    ordered = _node_sanitize(value)
+    ordered = _yaml.node_sanitize(value)
     string = pickle.dumps(ordered)
     return hashlib.sha256(string).hexdigest()
 
 
-# _node_sanitize()
-#
-# Returnes an alphabetically ordered recursive copy
-# of the source node with internal provenance information stripped.
-#
-# Only dicts are ordered, list elements are left in order.
-#
-def _node_sanitize(node):
-
-    if isinstance(node, collections.Mapping):
-
-        result = OrderedDict()
-
-        for key in sorted(node):
-            if key == _yaml.PROVENANCE_KEY:
-                continue
-            result[key] = _node_sanitize(node[key])
-
-        return result
-
-    elif isinstance(node, list):
-        return [_node_sanitize(elt) for elt in node]
-
-    return node
-
-
-def _node_chain_copy(source):
-    copy = collections.ChainMap({}, source)
-    for key, value in source.items():
-        if isinstance(value, collections.Mapping):
-            copy[key] = _node_chain_copy(value)
-        elif isinstance(value, list):
-            copy[key] = _list_chain_copy(value)
-        elif isinstance(value, _yaml.Provenance):
-            copy[key] = value.clone()
-
-    return copy
-
-
-def _list_chain_copy(source):
-    copy = []
-    for item in source:
-        if isinstance(item, collections.Mapping):
-            copy.append(_node_chain_copy(item))
-        elif isinstance(item, list):
-            copy.append(_list_chain_copy(item))
-        elif isinstance(item, _yaml.Provenance):
-            copy.append(item.clone())
-        else:
-            copy.append(item)
-
-    return copy
-
-
 # _set_deterministic_mtime()
 #
 # Set the mtime for every file in a directory tree to the same.
diff --git a/tests/yaml/yaml.py b/tests/yaml/yaml.py
index b496fe47c..ec632674f 100644
--- a/tests/yaml/yaml.py
+++ b/tests/yaml/yaml.py
@@ -2,7 +2,7 @@ import os
 import pytest
 from collections import Mapping
 
-from buildstream import _yaml, utils
+from buildstream import _yaml
 from buildstream import LoadError, LoadErrorReason
 from buildstream._yaml import CompositePolicy
 
@@ -197,7 +197,7 @@ def test_node_get(datafiles):
     assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
 
 
-# Really this is testing utils._node_chain_copy(), we want to
+# Really this is testing _yaml.node_chain_copy(), we want to
 # be sure that when using a ChainMap copy, compositing values
 # still preserves the original values in the copied dict.
 #
@@ -213,7 +213,7 @@ def test_composite_preserve_originals(datafiles):
 
     base = _yaml.load(filename)
     overlay = _yaml.load(overlayfile)
-    base_copy = utils._node_chain_copy(base)
+    base_copy = _yaml.node_chain_copy(base)
     _yaml.composite_dict(base_copy, overlay,
                          policy=CompositePolicy.OVERWRITE, typesafe=True)