element: Move __expand_splits to cythonbschubert/optimize-splits

__expand_splits is a somewhat intensive method, that represents >1%
of the runtime.

Moving it to cython makes it 5 times faster.

This also requires exporting some 'Variables' symbols as public.

5 files changed, 37 insertions, 22 deletions

diff --git a/setup.py b/setup.py
index c93cb44f3..daf10fff1 100755
--- a/setup.py
+++ b/setup.py
@@ -403,7 +403,7 @@ def register_cython_module(module_name, dependencies=None):
 BUILD_EXTENSIONS = []
 
 register_cython_module("buildstream.node")
-register_cython_module("buildstream._element")
+register_cython_module("buildstream._element", dependencies=["buildstream._variables"])
 register_cython_module("buildstream._loader._loader")
 register_cython_module("buildstream._loader.types", dependencies=["buildstream.node"])
 register_cython_module("buildstream._yaml", dependencies=["buildstream.node"])
diff --git a/src/buildstream/_element.pyx b/src/buildstream/_element.pyx
index cc0178a3c..67bf9a9e3 100644
--- a/src/buildstream/_element.pyx
+++ b/src/buildstream/_element.pyx
@@ -1,6 +1,9 @@
 from pyroaring import BitMap
 
+from .node cimport MappingNode, SequenceNode
 from .types import Scope
+from ._variables cimport Variables
+
 
 # FIXME: we should make those as enums consumable from Cython
 cdef SCOPE_ALL = Scope.ALL
@@ -85,3 +88,22 @@ def dependencies_for_targets(elements, scope):
 
     else:
         yield from elements
+
+
+def expand_splits(Variables variables, MappingNode element_public):
+    cdef MappingNode element_bst = element_public.get_mapping('bst', default={})
+    cdef MappingNode element_splits = element_bst.get_mapping('split-rules', default={})
+
+    cdef str domain
+    cdef SequenceNode split_nodes
+    cdef list splits
+
+    # Resolve any variables in the public split rules directly
+    for domain, split_nodes in element_splits.items():
+        splits = [
+            variables.subst((<str> split).strip())
+            for split in split_nodes.as_str_list()
+        ]
+        element_splits[domain] = splits
+
+    return element_public
\ No newline at end of file
diff --git a/src/buildstream/_variables.pxd b/src/buildstream/_variables.pxd
new file mode 100644
index 000000000..3b917cc68
--- /dev/null
+++ b/src/buildstream/_variables.pxd
@@ -0,0 +1,12 @@
+from .node cimport MappingNode
+
+
+cdef class Variables:
+
+    cdef MappingNode original
+    cdef dict _expstr_map
+    cdef public dict flat
+
+    cpdef str subst(self, str string)
+    cdef dict _resolve(self, MappingNode node)
+    cdef dict _flatten(self)
\ No newline at end of file
diff --git a/src/buildstream/_variables.pyx b/src/buildstream/_variables.pyx
index 470feddc9..63ef6040b 100644
--- a/src/buildstream/_variables.pyx
+++ b/src/buildstream/_variables.pyx
@@ -65,10 +65,6 @@ PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
 #
 cdef class Variables:
 
-    cdef MappingNode original
-    cdef dict _expstr_map
-    cdef public dict flat
-
     def __init__(self, MappingNode node):
         self.original = node
         self._expstr_map = self._resolve(node)
@@ -87,7 +83,7 @@ cdef class Variables:
     # Raises:
     #    LoadError, if the string contains unresolved variable references.
     #
-    def subst(self, str string):
+    cpdef str subst(self, str string):
         expstr = _parse_expstr(string)
 
         try:
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 6711a5c83..71c0601be 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -265,7 +265,7 @@ class Element(Plugin):
 
         # Grab public domain data declared for this instance
         unexpanded_public = self.__extract_public(meta)
-        self.__public = self.__expand_splits(unexpanded_public)
+        self.__public = _element.expand_splits(self.__variables, unexpanded_public)
         self.__dynamic_public = None
 
         # Collect the composited element configuration and
@@ -2662,21 +2662,6 @@ class Element(Plugin):
 
         return element_public
 
-    # Expand the splits in the public data using the Variables in the element
-    def __expand_splits(self, element_public):
-        element_bst = element_public.get_mapping('bst', default={})
-        element_splits = element_bst.get_mapping('split-rules', default={})
-
-        # Resolve any variables in the public split rules directly
-        for domain, splits in element_splits.items():
-            splits = [
-                self.__variables.subst(split.strip())
-                for split in splits.as_str_list()
-            ]
-            element_splits[domain] = splits
-
-        return element_public
-
     def __init_splits(self):
         bstdata = self.get_public_data('bst')
         splits = bstdata.get_mapping('split-rules')