Merge branch 'danielsilverstone-ct/gc-play' into 'master'

_stream.py, _project.py: Manage GC during pipeline load

See merge request BuildStream/buildstream!1164

3 files changed, 36 insertions, 0 deletions

diff --git a/buildstream/_project.py b/buildstream/_project.py
index 92ac8cab5..6df54f70f 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -18,6 +18,7 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 #        Tiago Gomes <tiago.gomes@codethink.co.uk>
 
+import gc
 import os
 from collections import OrderedDict
 from collections.abc import Mapping
@@ -352,6 +353,9 @@ class Project():
                                              ticker=None,
                                              fetch_subprojects=fetch_subprojects)
 
+        # Loading elements generates a lot of garbage, clear it now
+        gc.collect()
+
         with self._context.timed_activity("Resolving elements"):
             elements = [
                 Element._new_from_meta(meta)
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index caaa48908..b0fce3817 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -941,6 +941,7 @@ class Stream():
     #    (list of Element): The primary element selection
     #    (list of Element): The tracking element selection
     #
+    @utils._with_gc_disabled
     def _load(self, targets, track_targets, *,
               selection=PipelineSelection.NONE,
               track_selection=PipelineSelection.NONE,
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 844153706..a4e161ed4 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -23,6 +23,8 @@ Utilities
 
 import calendar
 import errno
+import functools
+import gc
 import hashlib
 import os
 import re
@@ -1280,3 +1282,32 @@ def _search_upward_for_files(directory, filenames):
             # i.e. we've reached the root of the filesystem
             return None, None
         directory = parent_dir
+
+
+# _with_gc_disabled()
+#
+# Decorate a function to disable the garbage collector across its execution.
+#
+# In general, disabling the garbage collector should be considered to be an
+# extreme action.  Only use this in carefully selected subsets of the code
+# where we generally create a lot more objects than we throw away.  For example
+# in loading the stream.
+#
+# Args:
+#    func (callable): The callable to disable the GC for
+#
+# Returns:
+#    (callable): The decorated callable
+#
+def _with_gc_disabled(func):
+    @functools.wraps(func)
+    def _gc_disabled(*args, **kwargs):
+        try:
+            gc.disable()
+            return func(*args, **kwargs)
+        finally:
+            gc.enable()
+            # Clean up to ensure we don't grow any more, freeing up room to be
+            # used by other objects during the course of running BuildStream.
+            gc.collect()
+    return _gc_disabled