_loader/loadelement: make LoadElement an extension class

This reduces by more than 4x the time spent in building and calculating
the dependencies of LoadElement.

2 files changed, 38 insertions, 23 deletions

diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index 22dfd2877..efcdb7ba7 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -31,6 +31,25 @@ from .types import Symbol
 from .types cimport Dependency
 
 
+# Dependency():
+#
+# A link from a LoadElement to its dependencies.
+#
+# Keeps a link to one of the current Element's dependencies, together with
+# its dependency type.
+#
+# Args:
+#    element (LoadElement): a LoadElement on which there is a dependency
+#    dep_type (str): the type of dependency this dependency link is
+cdef class LoadDependency:
+    cdef public LoadElement element
+    cdef public str dep_type
+
+    def __init__(self, LoadElement element, str dep_type):
+        self.element = element
+        self.dep_type = dep_type
+
+
 # LoadElement():
 #
 # A transient object breaking down what is loaded allowing us to
@@ -41,25 +60,21 @@ from .types cimport Dependency
 #    name (str): The element name
 #    loader (Loader): The Loader object for this element
 #
-class LoadElement():
-    # Dependency():
-    #
-    # A link from a LoadElement to its dependencies.
-    #
-    # Keeps a link to one of the current Element's dependencies, together with
-    # its dependency type.
-    #
-    # Args:
-    #    element (LoadElement): a LoadElement on which there is a dependency
-    #    dep_type (str): the type of dependency this dependency link is
-    class Dependency:
-        def __init__(self, element, dep_type):
-            self.element = element
-            self.dep_type = dep_type
+cdef class LoadElement:
 
     _counter = count()
 
-    def __init__(self, node, filename, loader):
+    cdef public _yaml.Node node
+    cdef public str name
+    cdef public str full_name
+    cdef public list dependencies
+    cdef public int node_id
+    cdef public bint meta_done
+    cdef public object _loader
+    # FIXME: Pyroaring exporting its symbols would be nice
+    cdef public object _dep_cache
+
+    def __init__(self, _yaml.Node node, str filename, object loader):
 
         #
         # Public members
@@ -67,7 +82,6 @@ class LoadElement():
         self.node = node        # The YAML node
         self.name = filename    # The element name
         self.full_name = None   # The element full name (with associated junction)
-        self.deps = None        # The list of Dependency objects
         self.meta_done = False  # If the MetaElement for this LoadElement is done
         self.node_id = next(self._counter)
 
@@ -112,14 +126,16 @@ class LoadElement():
     # Returns:
     #    (bool): True if this LoadElement depends on 'other'
     #
-    def depends(self, other):
+    def depends(self, LoadElement other):
         self._ensure_depends_cache()
         return other.node_id in self._dep_cache
 
     ###########################################
     #            Private Methods              #
     ###########################################
-    def _ensure_depends_cache(self):
+    cdef _ensure_depends_cache(self):
+        cdef LoadDependency dep
+        cdef LoadElement elt
 
         if self._dep_cache:
             return
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index e279501ff..d83f6bdb5 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -29,7 +29,7 @@ from .._profile import Topics, PROFILER
 from .._includes import Includes
 
 from .types import Symbol
-from .loadelement import LoadElement, _extract_depends_from_node
+from .loadelement import LoadElement, LoadDependency, _extract_depends_from_node
 from .metaelement import MetaElement
 from .metasource import MetaSource
 from ..types import CoreWarnings
@@ -122,7 +122,7 @@ class Loader():
         # to resolve potential circular dependencies between them
         dummy_target = LoadElement(_yaml.new_empty_node(), "", self)
         dummy_target.dependencies.extend(
-            LoadElement.Dependency(element, Symbol.RUNTIME)
+            LoadDependency(element, Symbol.RUNTIME)
             for element in target_elements
         )
 
@@ -315,8 +315,7 @@ class Loader():
                                             .format(dep.provenance))
 
                 # All is well, push the dependency onto the LoadElement
-                current_element[0].dependencies.append(
-                    LoadElement.Dependency(dep_element, dep.dep_type))
+                current_element[0].dependencies.append(LoadDependency(dep_element, dep.dep_type))
             else:
                 # We do not have any more dependencies to load for this
                 # element on the queue, report any invalid dep names