_variables: Cythonize _internal_expand

Move _variables.py to be a Cython module.

`_internal_expand` is a function that is called a lot in BuildStream.
It is also entirely isolated and easy to cythonize.

4 files changed, 39 insertions, 18 deletions

diff --git a/.pylintrc b/.pylintrc
index c47ef92cf..0dc625a46 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -3,7 +3,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=
+extension-pkg-whitelist=buildstream._variables
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
diff --git a/setup.py b/setup.py
index fe977c123..90aa74d31 100755
--- a/setup.py
+++ b/setup.py
@@ -391,6 +391,7 @@ def register_cython_module(module_name, dependencies=None):
 
 BUILD_EXTENSIONS = []
 
+register_cython_module("buildstream._variables")
 
 #####################################################
 #             Main setup() Invocation               #
diff --git a/src/buildstream/_variables.py b/src/buildstream/_variables.pyx
index dc20a60c3..405171b61 100644
--- a/src/buildstream/_variables.py
+++ b/src/buildstream/_variables.pyx
@@ -18,6 +18,7 @@
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 #        Daniel Silverstone <daniel.silverstone@codethink.co.uk>
+#        Benjamin Schubert <bschubert@bloomberg.net>
 
 import re
 import sys
@@ -219,6 +220,37 @@ def _parse_expstr(instr):
         return PARSE_CACHE[instr]
 
 
+# Helper to expand recursively an expansion string in the context
+# of the given dictionary of expansion string
+#
+# Args:
+#     content (dict): dictionary context for resolving the variables
+#     value (list): expansion string to expand
+#     acc (list): list in which to add the result
+#     counter (int): counter to count the number of recursion in order to
+#                    detect cycles.
+#
+# Raises:
+#     KeyError: if any expansion is missing
+#     RecursionError: if a variable is defined recursively
+#
+cdef void _expand_expstr_helper(dict content, list value, list acc, int counter = 0) except *:
+    cdef Py_ssize_t idx = 0
+    cdef Py_ssize_t value_len = len(value)
+
+    if counter > 1000:
+        raise RecursionError()
+
+    while idx < value_len:
+        acc.append(value[idx])
+        idx += 1
+
+        if idx < value_len:
+            _expand_expstr_helper(content, <list> content[value[idx]], acc, counter + 1)
+
+        idx += 1
+
+
 # Helper to expand a given top level expansion string tuple in the context
 # of the given dictionary of expansion strings.
 #
@@ -233,19 +265,6 @@ def _expand_expstr(content, topvalue):
     if len(topvalue) == 2 and topvalue[0] == "":
         return _expand_expstr(content, content[topvalue[1]])
 
-    # Otherwise process fully...
-    def internal_expand(value):
-        idx = 0
-        value_len = len(value)
-
-        while idx < value_len:
-            # First yield any constant string content
-            yield value[idx]
-            idx += 1
-            # Now, if there is an expansion variable left to expand, yield
-            # the expansion of that variable too
-            if idx < value_len:
-                yield from internal_expand(content[value[idx]])
-            idx += 1
-
-    return "".join(internal_expand(topvalue))
+    cdef list result = []
+    _expand_expstr_helper(content, topvalue, result)
+    return "".join(result)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 08326c6f3..8e006ea6b 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -931,7 +931,8 @@ class Element(Plugin):
            (str): The resolved value for *varname*, or None if no
            variable was declared with the given name.
         """
-        return self.__variables.flat.get(varname)
+        # Flat is not recognized correctly by Pylint as being a dictionary
+        return self.__variables.flat.get(varname)  # pylint: disable=no-member
 
     def batch_prepare_assemble(self, flags, *, collect=None):
         """ Configure command batching across prepare() and assemble()